From ac87028e24df88fd8096571ad480c36f0254b3f1 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Tue, 20 Feb 2024 12:01:19 +0100 Subject: [PATCH] [devops] Don't allow re-running just the Windows tests job. (#20116) Don't allow re-running just the Windows tests job, because the Mac that was reserved for us isn't reserved anymore. Instead all the jobs must be re-run. Accomplish this by storing the current BuildId in a file on the Mac bot, and then verifying that it's the expeted BuildId from the test step from Windows. --- .../scripts/clean-for-remote-tests.sh | 14 ++++++++ .../scripts/prepare-for-remote-tests.sh | 13 -------- .../automation/templates/windows/build.yml | 16 +++++++++ .../templates/windows/reenable-mac.yml | 33 ++++++++++++++++++- .../templates/windows/reserve-mac.yml | 8 +++++ .../automation/templates/windows/stage.yml | 1 + 6 files changed, 71 insertions(+), 14 deletions(-) create mode 100755 tools/devops/automation/scripts/clean-for-remote-tests.sh diff --git a/tools/devops/automation/scripts/clean-for-remote-tests.sh b/tools/devops/automation/scripts/clean-for-remote-tests.sh new file mode 100755 index 0000000000..c41d164623 --- /dev/null +++ b/tools/devops/automation/scripts/clean-for-remote-tests.sh @@ -0,0 +1,14 @@ +#!/bin/bash -eux + +# I've seen machines with more than 1gb of Xamarin.Messaging logs, so clean that up. +if du -hs ~/Library/Logs/Xamarin.Messaging*; then + rm -rf ~/Library/Logs/Xamarin.Messaging* +fi + +# Make sure we don't have any old stuff installed +if du -hs ~/Library/Caches/Xamarin; then + rm -rf ~/Library/Caches/Xamarin +fi + +# Make sure we don't have stuff from earlier builds. +rm -rf ~/remote_build_testing diff --git a/tools/devops/automation/scripts/prepare-for-remote-tests.sh b/tools/devops/automation/scripts/prepare-for-remote-tests.sh index b38900256a..a24cd9355b 100755 --- a/tools/devops/automation/scripts/prepare-for-remote-tests.sh +++ b/tools/devops/automation/scripts/prepare-for-remote-tests.sh @@ -1,18 +1,5 @@ #!/bin/bash -eux -# I've seen machines with more than 1gb of Xamarin.Messaging logs, so clean that up. -if du -hs ~/Library/Logs/Xamarin.Messaging*; then - rm -rf ~/Library/Logs/Xamarin.Messaging* -fi - -# Make sure we don't have any old stuff installed -if du -hs ~/Library/Caches/Xamarin; then - rm -rf ~/Library/Caches/Xamarin -fi - -# Make sure we don't have stuff from earlier builds. -rm -rf ~/remote_build_testing - # Install the local .NET we're using into XMA's directory # (we can't point XMA to our local directory) mkdir -p ~/Library/Caches/Xamarin/XMA/SDKs diff --git a/tools/devops/automation/templates/windows/build.yml b/tools/devops/automation/templates/windows/build.yml index 3a81a9a318..a5d5c601ff 100644 --- a/tools/devops/automation/templates/windows/build.yml +++ b/tools/devops/automation/templates/windows/build.yml @@ -96,7 +96,23 @@ steps: displayName: 'Show Environment' - pwsh: | + Import-Module $Env:SYSTEM_DEFAULTWORKINGDIRECTORY\\xamarin-macios\\tools\\devops\\automation\\scripts\\MaciosCI.psd1 ssh -v -i "$(ID_RSA_PATH)" -o IdentitiesOnly=yes -o StrictHostKeyChecking=no builder@$Env:MAC_AGENT_IP pwd + Invoke-SshDownload ` + -RemoteHost "$Env:MAC_AGENT_IP" ` + -RemoteUserName "$Env:MAC_AGENT_USER" ` + -Source "/Users/$Env:MAC_AGENT_USER/remote_build_testing/BuildId.txt" ` + -Target "BuildId.txt" + + $macBuildId = (Get-Content -Path BuildId.txt).Trim() + Remove-Item -Path BuildId.txt # clean up after ourselves + if ([string]::IsNullOrEmpty($macBuildId)) { + throw "The mac we're trying to connect to ($($Env:MAC_AGENT_IP)) is not reserved for us anymore. If you're reran this job, please rerun all jobs instead." + } + $windowsBuildId = $Env:BUILD_BUILDID + if ($macBuildId -ne $windowsBuildId) { + throw "The mac we're trying to connect to ($($Env:MAC_AGENT_IP)) is reserved for a different build (with build id $($macBuildId)). If you're reran this job, please rerun all jobs instead." + } displayName: "Verify ssh connection" # This task fixes errors such as these: diff --git a/tools/devops/automation/templates/windows/reenable-mac.yml b/tools/devops/automation/templates/windows/reenable-mac.yml index 25d51f4609..1c5d6245c8 100644 --- a/tools/devops/automation/templates/windows/reenable-mac.yml +++ b/tools/devops/automation/templates/windows/reenable-mac.yml @@ -22,13 +22,44 @@ steps: - pwsh: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/show_env.ps1 displayName: 'Dump Environment' +- task: AzureKeyVault@2 + inputs: + azureSubscription: 'Xamarin - R&D - XamarinSecurity' + KeyVaultName: 'xamarin-ios-vault' + SecretsFilter: 'RemoteMacIdRsa' + displayName: 'Download id_rsa' + +- pwsh: | + $idRsaPath = "$(Get-Location)\id_rsa" + Write-Host "##vso[task.setvariable variable=ID_RSA_PATH]$idRsaPath" + Add-Content -Path "id_rsa" -Value "$(RemoteMacIdRsa)" + # We need to make sure the private key is only accessible by the current user, + # otherwise ssh will complain and not use it. + icacls id_rsa /inheritance:r + $grant="$Env:USERNAME" + ":(R)" + icacls id_rsa /grant:r $grant + displayName: "Write and verify id_rsa" + continueOnError: true + +- pwsh: | + ssh -v -i "$(ID_RSA_PATH)" -o IdentitiesOnly=yes -o StrictHostKeyChecking=no builder@$Env:MAC_AGENT_IP -- rm -f "/Users/$($Env:MAC_AGENT_USER)/remote_build_testing/BuildId.txt" + displayName: 'Remove BuildId from macOS bot' + condition: always() + continueOnError: true + - pwsh: | Import-Module $Env:SYSTEM_DEFAULTWORKINGDIRECTORY\xamarin-macios\tools\devops\automation\scripts\MaciosCI.psd1 $vsts = New-VSTS -Org "devdiv" -Project "DevDiv" -Token $(MacPoolAccessToken) - # get the pool and the agent objects and disable the bot + # get the pool and the agent objects and enable the bot $pool = $vsts.Pools.GetPool("$Env:MAC_AGENT_POOL") $agent = $vsts.Agents.GetAgent($pool, $Env:MAC_AGENT_NAME) $vsts.Agents.SetEnabled($pool, $agent, $True) displayName: 'Re-enabled macOS bot from pool' condition: always() + +- pwsh: | + Remove-Item "$(ID_RSA_PATH)" + displayName: "Remove secrets" + condition: always() + continueOnError: true diff --git a/tools/devops/automation/templates/windows/reserve-mac.yml b/tools/devops/automation/templates/windows/reserve-mac.yml index 2832ebbcdd..fe146c3159 100644 --- a/tools/devops/automation/templates/windows/reserve-mac.yml +++ b/tools/devops/automation/templates/windows/reserve-mac.yml @@ -43,6 +43,10 @@ steps: name: macInfo displayName: Set agent information +- bash: $(Build.SourcesDirectory)/xamarin-macios/tools/devops/automation/scripts/clean-for-remote-tests.sh + displayName: 'Clean for remote tests' + continueOnError: true + - bash: ./install-qa-provisioning-profiles.sh -v displayName: 'Add build provisioning profiles' timeoutInMinutes: 30 @@ -58,6 +62,10 @@ steps: - pwsh: | Import-Module $Env:SYSTEM_DEFAULTWORKINGDIRECTORY\xamarin-macios\tools\devops\automation\scripts\MaciosCI.psd1 + + New-Item -Path "$($Env:HOME)" -Name "remote_build_testing" -Force -ItemType "directory" + New-Item -Path "$($Env:HOME)/remote_build_testing" -Name "BuildId.txt" -ItemType "file" -Force -Value "$($Env:BUILD_BUILDID)" + $vsts = New-VSTS -Org "devdiv" -Project "DevDiv" -Token $(MacPoolAccessToken) # get the pool and the agent objects and disable the bot diff --git a/tools/devops/automation/templates/windows/stage.yml b/tools/devops/automation/templates/windows/stage.yml index 96a4cb877f..a88d00d9bb 100644 --- a/tools/devops/automation/templates/windows/stage.yml +++ b/tools/devops/automation/templates/windows/stage.yml @@ -130,3 +130,4 @@ stages: MAC_AGENT_NAME: $[ dependencies.mac_reservation.outputs['macInfo.AGENT_NAME'] ] MAC_AGENT_POOL: $[ dependencies.mac_reservation.outputs['macInfo.AGENT_POOL'] ] MAC_AGENT_IP: $[ dependencies.mac_reservation.outputs['macInfo.AGENT_IP'] ] + MAC_AGENT_USER: builder