Merge pull request #1112 from mjudeikis/versioned.tests

Run e2e tests from the image

.pipelines/deploy-azure-env.yml

@@ -3,17 +3,19 @@ trigger: none
 pr: none
 
 parameters:
-- name: location
-- name: vsoConfigBuildID
-- name: vsoDeployerBuildID
 - name: fullDeploy
   type: boolean
   default: false
+- name: location
+- name: vsoConfigBuildID
+- name: vsoDeployerBuildID
 
 jobs:
 - template: ./templates/template-job-deploy-azure-env.yml
   parameters:
     environment: RP-Prod
+    rpMode: ''
+    aroVersionStorageAccount: $(aro-version-storage-account)
     locations:
     - ${{ parameters.location }}
     configFileName: $(config-file-name)

.pipelines/e2e-with-billing.yml

@@ -16,6 +16,8 @@ stages:
         location: $(LOCATION)
         subscription: $(e2e-subscription)
         azureDevOpsE2EJSONSPN: $(aro-v4-e2e-devops-spn)
+        aroVersionStorageAccount: $(aro-version-storage-account)
+        rpMode: int
 - stage: Delay_For_Billing_Table
   displayName: Wait 6 hours for billing table ready
   jobs:

.pipelines/e2e.yml

@@ -21,8 +21,27 @@ jobs:
   - template: ./templates/template-push-images-to-acr.yml
     parameters:
       rpImageACR: $(RP_IMAGE_ACR)
-  - template: ./templates/template-az-cli-set-context.yml
-  - template: ./templates/template-deploy-e2e-db.yml
-  - template: ./templates/template-run-rp-and-e2e.yml
-  - template: ./templates/template-clean-e2e-db.yml
+  - script: |
+      set -e
+      set -o pipefail
+
+      . secrets/env
+
+      az account set -s $AZURE_SUBSCRIPTION_ID
+
+      set -x
+      . ./hack/e2e/run-rp-and-e2e.sh
+      trap 'set +e; kill_rp; clean_e2e_db' EXIT
+
+      deploy_e2e_db
+
+      run_rp
+      validate_rp_running
+      register_sub
+
+      export CI=true
+
+      # Don't override RESOURCEGROUP more widely because clean_e2e_db depends
+      # on the original value
+      RESOURCEGROUP=$ARO_RESOURCEGROUP make test-e2e
   - template: ./templates/template-az-cli-logout.yml

.pipelines/int-release.yml

@@ -11,7 +11,7 @@ parameters:
   default: latest
 - name: fullDeploy
   type: boolean
-  default: false
+  default: true
 
 stages:
 - stage: Deploy_INT
@@ -21,6 +21,8 @@ stages:
   - template: ./templates/template-job-deploy-azure-env.yml
     parameters:
       environment: RP-INT
+      rpMode: int
+      aroVersionStorageAccount: $(aro-version-storage-account)
       locations:
       - eastus
       configFileName: int-config.yaml

.pipelines/prod-release.yml

@@ -3,11 +3,11 @@ pr: none
 trigger: none
 
 parameters:
-- name: vsoConfigBuildID
-- name: vsoDeployerBuildID
 - name: fullDeploy
   type: boolean
   default: false
+- name: vsoConfigBuildID
+- name: vsoDeployerBuildID
 
 stages:
 - stage: Deploy_CanarySector
@@ -17,6 +17,8 @@ stages:
   - template: ./templates/template-job-deploy-azure-env.yml
     parameters:
       environment: RP-Prod-CanarySector
+      rpMode: ''
+      aroVersionStorageAccount: $(aro-version-storage-account)
       locations:
       - westcentralus
       - eastus2euap
@@ -40,6 +42,8 @@ stages:
   - template: ./templates/template-job-deploy-azure-env.yml
     parameters:
       environment: RP-Prod-LowTrafficSector
+      rpMode: ''
+      aroVersionStorageAccount: $(aro-version-storage-account)
       locations:
       - australiaeast
       - australiasoutheast
@@ -68,6 +72,8 @@ stages:
   - template: ./templates/template-job-deploy-azure-env.yml
     parameters:
       environment: RP-Prod-USSector
+      rpMode: ''
+      aroVersionStorageAccount: $(aro-version-storage-account)
       locations:
       - centralus
       - eastus
@@ -96,6 +102,8 @@ stages:
   - template: ./templates/template-job-deploy-azure-env.yml
     parameters:
       environment: RP-Prod-EuropeSector
+      rpMode: ''
+      aroVersionStorageAccount: $(aro-version-storage-account)
       locations:
       - canadacentral
       - canadaeast
@@ -126,6 +134,8 @@ stages:
   - template: ./templates/template-job-deploy-azure-env.yml
     parameters:
       environment: RP-Prod-ROWSector
+      rpMode: ''
+      aroVersionStorageAccount: $(aro-version-storage-account)
       locations:
       - brazilsouth
       - southeastasia

.pipelines/templates/template-az-cli-set-context.yml

@@ -1,9 +0,0 @@
-parameters:
-  azureDevOpsJSONSPN: ''
-steps:
-- script: |
-    . secrets/env
-    set -x
-    . ./hack/e2e/run-rp-and-e2e.sh
-    set_cli_context
-  displayName: ⚙️ AZ Context Setup

.pipelines/templates/template-clean-e2e-db.yml

@@ -1,10 +0,0 @@
-steps:
-- script: |
-    . secrets/env
-
-    set -x
-    . ./hack/e2e/run-rp-and-e2e.sh
-
-    clean_e2e_db
-  displayName: 🧹 Clean cluster DB
-  condition: and(always(), eq(variables['RP_MODE'], 'development'))

.pipelines/templates/template-clean-subscription.yml

@@ -1,10 +1,10 @@
 parameters:
+  dryRun: ''
   purgeTTL: ''
   purgeCreatedTag: ''
+  resourceGroupDeletePrefixes: ''
   subscriptionCredentialsJSON: ''
   subscriptionId: ''
-  dryRun: ''
-  resourceGroupDeletePrefixes: ''
 
 steps:
 - script: |

.pipelines/templates/template-deploy-azure-env.yml

@@ -1,10 +1,11 @@
 parameters:
-  location: ''
-  configDirectory: ''
-  deployerDirectory: ''
-  configFileName: ''
   azureDevOpsJSONSPN: ''
+  configDirectory: ''
+  configFileName: ''
+  deployerDirectory: ''
   fullDeploy: ''
+  location: ''
+
 steps:
 - script: |
     set -eu

.pipelines/templates/template-deploy-e2e-db.yml

@@ -1,10 +0,0 @@
-steps:
-- script: |
-    . secrets/env
-
-    set -x
-    . ./hack/e2e/run-rp-and-e2e.sh
-
-    deploy_e2e_db
-  displayName: 🚀 Deploy custom RP DB
-  condition: eq(variables['RP_MODE'], 'development')

.pipelines/templates/template-deploy-shared-env.yml

@@ -1,6 +1,7 @@
 parameters:
-  subscriptionID: ''
   azureDevOpsJSONSPN: ''
+  subscriptionID: ''
+
 steps:
 - script: |
     set -e

.pipelines/templates/template-job-deploy-azure-env.yml

@@ -1,19 +1,21 @@
 # Azure DevOps Job deploying rp
 parameters:
-  environment: ''
-  locations: []
-  configFileName: ''
+  aroVersionStorageAccount: ''
+  azureDevOpsE2EJSONSPN: ''
   azureDevOpsJSONSPN: ''
+  billingE2EPipelineName: ''
+  billingE2EBranchName: ''
+  configFileName: ''
+  e2eSubscription: ''
+  environment: ''
+  fullDeploy: ''
+  locations: []
+  rpMode: ''
   vsoProjectID: ''
   vsoConfigPipelineID: ''
   vsoConfigBuildID: ''
   vsoDeployerPipelineID: ''
   vsoDeployerBuildID: ''
-  azureDevOpsE2EJSONSPN: ''
-  e2eSubscription: ''
-  billingE2EPipelineName: ''
-  billingE2EBranchName: ''
-  fullDeploy: ''
 
 jobs:
 - ${{ each location in  parameters.locations }}:
@@ -76,6 +78,8 @@ jobs:
         location: ${{ location }}
         subscription: ${{ parameters.e2eSubscription }}
         azureDevOpsE2EJSONSPN: ${{ parameters.azureDevOpsE2EJSONSPN }}
+        aroVersionStorageAccount: ${{ parameters.aroVersionStorageAccount }}
+        rpMode: ${{ parameters.rpMode }}
   - job: Billing_E2E_${{ location }}_Wait
     dependsOn: RP_E2E_${{ location }}
     timeoutInMinutes: 400

.pipelines/templates/template-prod-e2e-steps.yml

@@ -1,29 +1,56 @@
 parameters:
-  location: ''
-  subscription: ''
+  aroVersionStorageAccount: ''
   azureDevOpsE2EJSONSPN: ''
+  location: ''
+  rpMode: ''
+  subscription: ''
 
 steps:
-# TODO(mj) we need to checkout code to run tooling like kubeconfig. This should go away once we refactor everything into golang.
-- template: ./template-checkout.yml
-- template: ./template-az-cli-login.yml
-  parameters:
-    azureDevOpsJSONSPN: ${{ parameters.azureDevOpsE2EJSONSPN }}
 - script: |
     set -e
     export LOCATION=${{ parameters.location }}
     export AZURE_SUBSCRIPTION_ID=${{ parameters.subscription }}
 
-    trap 'rm -f devops-spn.json' EXIT
+    trap 'set +e; rm -f devops-spn.json ; for c in $(docker ps -aq); do docker rm -f $c; done; docker image prune -af; rm -rf ~/.docker/config.json' EXIT
     base64 -d >devops-spn.json <<<${{ parameters.azureDevOpsE2EJSONSPN }}
     export AZURE_CLIENT_ID=$(jq -r .clientId <devops-spn.json)
     export AZURE_CLIENT_SECRET=$(jq -r .clientSecret <devops-spn.json)
     export AZURE_TENANT_ID=$(jq -r .tenantId <devops-spn.json)
 
-    set -x
-    . ./hack/e2e/run-rp-and-e2e.sh
 
-    export CI=true
-    run_e2e
+    # TODO: read RP config and derive $IMAGE from it. This would also need the
+    # AZURE_CLIENT_ID identity to be able to pull from non-INT ACR.
+    export IMAGE=arointsvc.azurecr.io/aro
+    docker login $IMAGE -u $AZURE_CLIENT_ID -p $AZURE_CLIENT_SECRET
+
+    set -x
+
+    VERSION=$(curl -sf "https://${{ parameters.aroVersionStorageAccount }}.blob.core.windows.net/rpversion/$LOCATION")
+
+    # TODO: consider removing E2E_CREATE_CLUSTER, E2E_DELETE_CLUSTER below after next RP
+    # deployment.
+
+    # TODO: Remove AZURE_FP_CLIENT_ID once RP is deployed in INT
+    export AZURE_FP_CLIENT_ID="71cfb175-ea3a-444e-8c03-b119b2752ce4"
+
+    # TODO: e2e.test arguments need to move inside the container somehow.  Maybe
+    # a short script inside the container?  :-|
+    docker run \
+      --rm \
+      --env CI=true \
+      --env RP_MODE=${{ parameters.rpMode }} \
+      --env E2E_CREATE_CLUSTER=true \
+      --env AZURE_FP_CLIENT_ID \
+      --env E2E_DELETE_CLUSTER=true \
+      --env AZURE_CLIENT_ID \
+      --env AZURE_CLIENT_SECRET \
+      --env AZURE_TENANT_ID \
+      --env AZURE_SUBSCRIPTION_ID \
+      --env LOCATION \
+      --env RESOURCEGROUP="v4-e2e-V$BUILD_BUILDID-$LOCATION" \
+      --env CLUSTER="v4-e2e-V$BUILD_BUILDID-$LOCATION" \
+      --entrypoint e2e.test \
+      "$IMAGE:$VERSION" \
+      -test.timeout 180m -test.v -ginkgo.v
+
   displayName: 🚀 Run ${{ parameters.location }} E2E
-- template: ./template-az-cli-logout.yml

.pipelines/templates/template-push-images-to-acr.yml

@@ -3,7 +3,7 @@ parameters:
 steps:
 - script: |
     set -e
-    trap 'for c in $(docker ps -aq); do docker rm -f $c; done; docker image prune -af' EXIT
+    trap 'set +e; for c in $(docker ps -aq); do docker rm -f $c; done; docker image prune -af ; rm -rf ~/.docker/config.json' EXIT
 
     export RP_IMAGE_ACR=${{ parameters.rpImageACR }}
 

.pipelines/templates/template-rp-and-billing-e2e-section.yml

@@ -1,9 +1,9 @@
 parameters:
-  last_stage_of_previous_section: ''
-  e2e_subscription: ''
   aro_v4_e2e_devops_spn: ''
   billing_e2e_pipeline_name: ''
   billing_e2e_branch_name: ''
+  e2e_subscription: ''
+  last_stage_of_previous_section: ''
   locations: []
 
 stages:

.pipelines/templates/template-rp-and-billing-e2e.yml

@@ -1,11 +1,11 @@
 # Azure DevOps Pipeline running RP e2e and Billing e2e
 parameters:
-  sign_off_stage: ''
-  e2e_subscription: ''
   aro_v4_e2e_devops_spn: ''
   billing_e2e_pipeline_name: ''
   billing_e2e_branch_name: ''
+  e2e_subscription: ''
   location: ''
+  sign_off_stage: ''
 
 stages:
 - stage: ${{ parameters.location }}_RP_E2E
@@ -22,6 +22,8 @@ stages:
         location: ${{ parameters.location }}
         subscription: ${{ parameters.e2e_subscription }}
         azureDevOpsE2EJSONSPN: ${{ parameters.aro_v4_e2e_devops_spn }}
+        aroVersionStorageAccount: $(aro-version-storage-account)
+        rpMode: ''
 - stage: ${{ parameters.location }}_Delay_For_Billing_Table
   displayName: ${{ parameters.location }} wait 6 hours for billing table ready
   jobs:

.pipelines/templates/template-run-rp-and-e2e.yml

@@ -1,25 +0,0 @@
-steps:
-- script: |
-    set -e
-    set -o pipefail
-
-    . secrets/env
-
-    set -x
-    . ./hack/e2e/run-rp-and-e2e.sh
-
-    if [ $RP_MODE = "development" ]
-    then
-      run_rp
-      validate_rp_running
-      register_sub
-    fi
-
-    export CI=true
-    run_e2e
-
-    if [ $RP_MODE = "development" ]
-    then
-      kill_rp
-    fi
-  displayName: 🚀 Run ARO RP & E2E

docs/prepare-a-shared-rp-development-environment.md

@@ -277,7 +277,6 @@ locations.
    EOF
    ```
 
-
 ## Deploy shared RP development environment (once per location)
 
 Look at the [helper file](../hack/devtools/deploy-shared-env.sh) to understand

docs/testing.md

@@ -62,7 +62,7 @@ validate_rp_running
 register_sub
 
 # Run e2e
-run_e2e
+make test-e2e
 
 # Stop the local RP
 kill_rp

hack/e2e/run-rp-and-e2e.sh

@@ -14,9 +14,9 @@ validate_rp_running() {
             *)
             echo "Attempt $ELAPSED - local RP is NOT up. Code : $http_code, waiting"
             sleep 2
-            # after 20 secs return exit 1 to not block ci
+            # after 40 secs return exit 1 to not block ci
             ELAPSED=$((ELAPSED+1))
-            if [ $ELAPSED -eq 10 ]
+            if [ $ELAPSED -eq 20 ]
             then
                 exit 1
             fi
@@ -51,11 +51,6 @@ deploy_e2e_db() {
 
 }
 
-set_cli_context() {
-    echo "########## Setting az cli context ##########"
-    az account set -s $AZURE_SUBSCRIPTION_ID
-}
-
 register_sub() {
     echo "########## 🔑 Registering subscription ##########"
     curl -sko /dev/null -X PUT \
@@ -64,10 +59,6 @@ register_sub() {
       "https://localhost:8443/subscriptions/$AZURE_SUBSCRIPTION_ID?api-version=2.0"
 }
 
-run_e2e() {
-    RESOURCEGROUP=$ARO_RESOURCEGROUP make test-e2e
-}
-
 clean_e2e_db(){
     echo "########## 🧹 Deleting DB $DATABASE_NAME ##########"
     az cosmosdb sql database delete --name $DATABASE_NAME \
@@ -81,8 +72,6 @@ clean_e2e_db(){
 export CLUSTER="v4-e2e-V$BUILD_BUILDID-$LOCATION"
 export DATABASE_NAME="v4-e2e-V$BUILD_BUILDID-$LOCATION"
 export ARO_RESOURCEGROUP="v4-e2e-V$BUILD_BUILDID-$LOCATION"
-export E2E_CREATE_CLUSTER=true
-export E2E_DELETE_CLUSTER=true
 
 echo "######################################"
 echo "##### ARO V4 E2e helper sourced ######"

pkg/util/cluster/cluster.go

@@ -53,6 +53,11 @@ type Cluster struct {
 	roleassignments   authorization.RoleAssignmentsClient
 }
 
+const (
+	firstPartyClientIDProduction  = "f1dd0a37-89c6-4e07-bcd1-ffd3d43d8875"
+	firstPartyClientIDIntegration = "71cfb175-ea3a-444e-8c03-b119b2752ce4"
+)
+
 type errors []error
 
 func (errs errors) Error() string {
@@ -67,7 +72,7 @@ func (errs errors) Error() string {
 }
 
 func New(log *logrus.Entry, deploymentMode deployment.Mode, instancemetadata instancemetadata.InstanceMetadata, ci bool) (*Cluster, error) {
-	if deploymentMode != deployment.Production {
+	if deploymentMode == deployment.Development {
 		for _, key := range []string{
 			"AZURE_FP_CLIENT_ID",
 		} {
@@ -112,8 +117,13 @@ func (c *Cluster) Create(ctx context.Context, clusterName string) error {
 		return nil
 	}
 
-	fpClientID := "f1dd0a37-89c6-4e07-bcd1-ffd3d43d8875"
-	if c.deploymentMode != deployment.Production {
+	var fpClientID string
+	switch c.deploymentMode {
+	case deployment.Integration:
+		fpClientID = firstPartyClientIDIntegration
+	case deployment.Production:
+		fpClientID = firstPartyClientIDProduction
+	default:
 		fpClientID = os.Getenv("AZURE_FP_CLIENT_ID")
 	}
 

test/e2e/setup.go

@@ -149,7 +149,7 @@ func setup(ctx context.Context) error {
 
 	clusterName = os.Getenv("CLUSTER")
 
-	if os.Getenv("E2E_CREATE_CLUSTER") != "" {
+	if os.Getenv("CI") != "" { // always create cluster in CI
 		cluster, err := cluster.New(log, deploymentMode, im, os.Getenv("CI") != "")
 		if err != nil {
 			return err
@@ -170,7 +170,8 @@ func setup(ctx context.Context) error {
 }
 
 func done(ctx context.Context) error {
-	if os.Getenv("E2E_DELETE_CLUSTER") != "" {
+	// terminate early if delete flag is set to false
+	if os.Getenv("CI") != "" && os.Getenv("E2E_DELETE_CLUSTER") != "false" {
 		cluster, err := cluster.New(log, deploymentMode, im, os.Getenv("CI") != "")
 		if err != nil {
 			return err