Basic remote job on azure pipelines (#4657)

.azure-pipelines-templates/test_on_remote.yml

@@ -0,0 +1,31 @@
+parameters:
+  depends_on: ""
+  condition: ""
+
+jobs:
+  - job: ${{ parameters.job_name }}
+    displayName: ${{ parameters.display_name }}
+    dependsOn: ${{ parameters.depends_on }}
+    condition: ${{ parameters.condition }}
+    pool:
+      vmImage: ubuntu-20.04
+    timeoutInMinutes: 120
+    variables:
+      RUN_ON: ${{ parameters.run_on }}
+      HOST_PRIVATE_KEY: ${{ parameters.host_private_key }}
+
+    steps:
+      - script: |
+          mkdir ~/.ssh
+          echo $(HOST_PRIVATE_KEY) | base64 -d > ~/.ssh/id_rsa
+          sudo chmod 600 ~/.ssh/id_rsa
+        name: setup_key
+
+      - script: |
+          echo -e "Testing connection from separate job..."
+          IFS='\n' read -ra IP_ADDR_LIST <<< "$(RUN_ON)"
+          for IP_ADDR in "${IP_ADDR_LIST[@]}"; do
+            echo -e "Testing connection with $IP_ADDR"
+            ssh agent@$IP_ADDR -o "StrictHostKeyChecking no" 'echo "Connected successfully"'
+          done
+        name: test_connection

.azure_pipelines_snp.yml

@@ -33,16 +33,27 @@ jobs:
       vmImage: ubuntu-20.04
     steps:
       - script: |
+          echo -e "\nGenerating an SSH key for use with the executors..."
+          ssh-keygen -t rsa -b 4096 -f ~/.ssh/id_rsa -N ""
+          PUB_KEY=`cat ~/.ssh/id_rsa.pub`
+          PRIV_KEY=`base64 -w 0 ~/.ssh/id_rsa`
+          echo "##vso[task.setvariable variable=hostPrivKey;isOutput=true;issecret=true]$PRIV_KEY"
+          echo -e "\nDone"
           echo -e "\nInstalling Azure CLI..."
           curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
           az login --service-principal -u $(CCF_SNP_CI_APP_ID) -p $(CCF_SNP_CI_SERVICE_PRINCIPAL_PASSWORD) --tenant $(CCF_SNP_CI_TENANT)
           pip install azure-mgmt-resource azure-identity azure-mgmt-containerinstance
           echo -e "\nInstalled Azure CLI and logged in"
           echo -e "\nDeploying Azure Container Instance/s..."
-          python3.8 scripts/azure_deployment/arm_template.py deploy aci --subscription-id $(CCF_AZURE_SUBSCRIPTION_ID) --resource-group ccf-aci --aci-type dynamic-agent --deployment-name ci-$(Build.BuildNumber) > ~/aci_ips
-          echo "##vso[task.setvariable variable=aciIp;isOutput=true]`cat ~/aci_ips`"
+          python3.8 scripts/azure_deployment/arm_template.py deploy aci --subscription-id $(CCF_AZURE_SUBSCRIPTION_ID) --resource-group ccf-aci --aci-type dynamic-agent --deployment-name ci-$(Build.BuildNumber) --aci-ssh-keys "$PUB_KEY" > ~/aci_ips
+          IP_ADDRESSES=`cat ~/aci_ips`
+          echo "##vso[task.setvariable variable=aciIp;isOutput=true]$IP_ADDRESSES"
+          IFS='\n' read -ra IP_ADDR_LIST <<< "$IP_ADDRESSES"
+          for IP_ADDR in "${IP_ADDR_LIST[@]}"; do
+            echo -e "Testing connection with $IP_ADDR"
+            for i in {1..3}; do ssh agent@$IP_ADDR -o "StrictHostKeyChecking no" 'echo "Connected successfully"' && break || echo "Connection failed, ACI may still be spinning up, retrying"; done
+          done
           echo -e "\nDone"
-          echo -e `cat ~/aci_ips`
         env:
           CCF_SNP_CI_APP_ID: $(CCF_SNP_CI_APP_ID)
           CCF_SNP_CI_SERVICE_PRINCIPAL_PASSWORD: $(CCF_SNP_CI_SERVICE_PRINCIPAL_PASSWORD)
@@ -50,11 +61,19 @@ jobs:
           CCF_AZURE_SUBSCRIPTION_ID: $(CCF_AZURE_SUBSCRIPTION_ID)
         name: deploy
 
+  - template: .azure-pipelines-templates/test_on_remote.yml
+    parameters:
+      job_name: test_snp
+      display_name: "Test SNP"
+      depends_on: deploy_aci
+      run_on: $[ dependencies.deploy_aci.outputs['deploy.aciIp'] ]
+      host_private_key: $[ dependencies.deploy_aci.outputs['deploy.hostPrivKey'] ]
+
   - job: cleanup_aci
     displayName: "Cleanup ACI"
     pool:
       vmImage: ubuntu-20.04
-    dependsOn: deploy_aci
+    dependsOn: test_snp
     condition: always()
     steps:
       - script: |