[devops] Don't allow re-running just the Windows tests job. (#20116)

Don't allow re-running just the Windows tests job, because the Mac that was
reserved for us isn't reserved anymore.

Instead all the jobs must be re-run.

Accomplish this by storing the current BuildId in a file on the Mac bot, and
then verifying that it's the expeted BuildId from the test step from Windows.

tools/devops/automation/scripts/clean-for-remote-tests.sh

@@ -0,0 +1,14 @@
+#!/bin/bash -eux
+
+# I've seen machines with more than 1gb of Xamarin.Messaging logs, so clean that up.
+if du -hs ~/Library/Logs/Xamarin.Messaging*; then
+	rm -rf ~/Library/Logs/Xamarin.Messaging*
+fi
+
+# Make sure we don't have any old stuff installed
+if du -hs ~/Library/Caches/Xamarin; then
+	rm -rf ~/Library/Caches/Xamarin
+fi
+
+# Make sure we don't have stuff from earlier builds.
+rm -rf ~/remote_build_testing

tools/devops/automation/scripts/prepare-for-remote-tests.sh

@@ -1,18 +1,5 @@
 #!/bin/bash -eux
 
-# I've seen machines with more than 1gb of Xamarin.Messaging logs, so clean that up.
-if du -hs ~/Library/Logs/Xamarin.Messaging*; then
-	rm -rf ~/Library/Logs/Xamarin.Messaging*
-fi
-
-# Make sure we don't have any old stuff installed
-if du -hs ~/Library/Caches/Xamarin; then
-	rm -rf ~/Library/Caches/Xamarin
-fi
-
-# Make sure we don't have stuff from earlier builds.
-rm -rf ~/remote_build_testing
-
 # Install the local .NET we're using into XMA's directory
 # (we can't point XMA to our local directory)
 mkdir -p ~/Library/Caches/Xamarin/XMA/SDKs

tools/devops/automation/templates/windows/build.yml

@@ -96,7 +96,23 @@ steps:
   displayName: 'Show Environment'
 
 - pwsh: |
+    Import-Module $Env:SYSTEM_DEFAULTWORKINGDIRECTORY\\xamarin-macios\\tools\\devops\\automation\\scripts\\MaciosCI.psd1
     ssh -v -i "$(ID_RSA_PATH)" -o IdentitiesOnly=yes -o StrictHostKeyChecking=no builder@$Env:MAC_AGENT_IP pwd
+    Invoke-SshDownload `
+      -RemoteHost "$Env:MAC_AGENT_IP" `
+      -RemoteUserName "$Env:MAC_AGENT_USER" `
+      -Source "/Users/$Env:MAC_AGENT_USER/remote_build_testing/BuildId.txt" `
+      -Target "BuildId.txt"
+
+    $macBuildId = (Get-Content -Path BuildId.txt).Trim()
+    Remove-Item -Path BuildId.txt # clean up after ourselves
+    if ([string]::IsNullOrEmpty($macBuildId)) {
+      throw "The mac we're trying to connect to ($($Env:MAC_AGENT_IP)) is not reserved for us anymore. If you're reran this job, please rerun all jobs instead."
+    }
+    $windowsBuildId = $Env:BUILD_BUILDID
+    if ($macBuildId -ne $windowsBuildId) {
+      throw "The mac we're trying to connect to ($($Env:MAC_AGENT_IP)) is reserved for a different build (with build id $($macBuildId)). If you're reran this job, please rerun all jobs instead."
+    }
   displayName: "Verify ssh connection"
 
 # This task fixes errors such as these:

tools/devops/automation/templates/windows/reenable-mac.yml

@@ -22,13 +22,44 @@ steps:
 - pwsh: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/show_env.ps1
   displayName: 'Dump Environment'
 
+- task: AzureKeyVault@2
+  inputs:
+    azureSubscription: 'Xamarin - R&D - XamarinSecurity'
+    KeyVaultName: 'xamarin-ios-vault'
+    SecretsFilter: 'RemoteMacIdRsa'
+  displayName: 'Download id_rsa'
+
+- pwsh: |
+    $idRsaPath = "$(Get-Location)\id_rsa"
+    Write-Host "##vso[task.setvariable variable=ID_RSA_PATH]$idRsaPath"
+    Add-Content -Path "id_rsa"  -Value "$(RemoteMacIdRsa)"
+    # We need to make sure the private key is only accessible by the current user,
+    # otherwise ssh will complain and not use it.
+    icacls id_rsa /inheritance:r
+    $grant="$Env:USERNAME" + ":(R)"
+    icacls id_rsa /grant:r $grant
+  displayName: "Write and verify id_rsa"
+  continueOnError: true
+
+- pwsh: |
+    ssh -v -i "$(ID_RSA_PATH)" -o IdentitiesOnly=yes -o StrictHostKeyChecking=no builder@$Env:MAC_AGENT_IP -- rm -f "/Users/$($Env:MAC_AGENT_USER)/remote_build_testing/BuildId.txt"
+  displayName: 'Remove BuildId from macOS bot'
+  condition: always()
+  continueOnError: true
+
 - pwsh: |
     Import-Module $Env:SYSTEM_DEFAULTWORKINGDIRECTORY\xamarin-macios\tools\devops\automation\scripts\MaciosCI.psd1
     $vsts = New-VSTS -Org "devdiv" -Project "DevDiv" -Token $(MacPoolAccessToken)
 
-    # get the pool and the agent objects and disable the bot
+    # get the pool and the agent objects and enable the bot
     $pool = $vsts.Pools.GetPool("$Env:MAC_AGENT_POOL")
     $agent = $vsts.Agents.GetAgent($pool, $Env:MAC_AGENT_NAME)
     $vsts.Agents.SetEnabled($pool, $agent, $True)
   displayName: 'Re-enabled macOS bot from pool'
   condition: always()
+
+- pwsh: |
+    Remove-Item "$(ID_RSA_PATH)"
+  displayName: "Remove secrets"
+  condition: always()
+  continueOnError: true

tools/devops/automation/templates/windows/reserve-mac.yml

@@ -43,6 +43,10 @@ steps:
   name: macInfo
   displayName: Set agent information
 
+- bash: $(Build.SourcesDirectory)/xamarin-macios/tools/devops/automation/scripts/clean-for-remote-tests.sh
+  displayName: 'Clean for remote tests'
+  continueOnError: true
+
 - bash: ./install-qa-provisioning-profiles.sh -v
   displayName: 'Add build provisioning profiles'
   timeoutInMinutes: 30
@@ -58,6 +62,10 @@ steps:
 
 - pwsh: |
     Import-Module $Env:SYSTEM_DEFAULTWORKINGDIRECTORY\xamarin-macios\tools\devops\automation\scripts\MaciosCI.psd1
+
+    New-Item -Path "$($Env:HOME)" -Name "remote_build_testing" -Force -ItemType "directory"
+    New-Item -Path "$($Env:HOME)/remote_build_testing" -Name "BuildId.txt" -ItemType "file" -Force -Value "$($Env:BUILD_BUILDID)"
+
     $vsts = New-VSTS -Org "devdiv" -Project "DevDiv" -Token $(MacPoolAccessToken)
 
     # get the pool and the agent objects and disable the bot

tools/devops/automation/templates/windows/stage.yml

@@ -130,3 +130,4 @@ stages:
       MAC_AGENT_NAME: $[ dependencies.mac_reservation.outputs['macInfo.AGENT_NAME'] ]
       MAC_AGENT_POOL: $[ dependencies.mac_reservation.outputs['macInfo.AGENT_POOL'] ]
       MAC_AGENT_IP: $[ dependencies.mac_reservation.outputs['macInfo.AGENT_IP'] ]
+      MAC_AGENT_USER: builder