From 7c37667c65ae6d1d29a6e7da617e6e7dc04528de Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 4 Dec 2024 02:58:16 +1100 Subject: [PATCH] Split out some env vars into their own section in the Compose file (#3978) * split out some env vars into their own section * env var fixing * some debug * because run-rp-and-e2e sources the env, this sourcing doesn't do anything * this doesn't do anything in this step * move the docker stuff into a different section * add in the args properly in docker-compose * fix typos * just use quiet pull instead --- .pipelines/ci.yml | 25 ++++-- docker-compose.yml | 157 +++++++++++++++++-------------------- hack/e2e/run-rp-and-e2e.sh | 3 + 3 files changed, 92 insertions(+), 93 deletions(-) diff --git a/.pipelines/ci.yml b/.pipelines/ci.yml index 4220d150b..173003fea 100644 --- a/.pipelines/ci.yml +++ b/.pipelines/ci.yml @@ -134,12 +134,25 @@ stages: parameters: azureDevOpsJSONSPN: $(aro-v4-e2e-devops-spn) - # Get Kubeconfig for AKS Cluster with corrected Key Vault configuration - bash: | az account set -s $AZURE_SUBSCRIPTION_ID SECRET_SA_ACCOUNT_NAME=$(SECRET_SA_ACCOUNT_NAME) make secrets + displayName: Fetch secrets + + - bash: | + echo "##vso[task.setvariable variable=CI]true" + displayName: Set CI=true + + # Override the E2E label for IndividualCI/BatchedCI (i.e. not manually + # ran/PR jobs) to run all non-smoke tasks (default is !smoke&&!regressiontest) + - bash: | + echo "##vso[task.setvariable variable=E2E_LABEL]!smoke" + displayName: Enable regression tests in CI + condition: in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI') + + # Get Kubeconfig for AKS Cluster with corrected Key Vault configuration + - bash: | . secrets/env - export KEYVAULT_PREFIX="e2e-classic-eastus-cls" # Retrieve the kubeconfig hack/get-admin-aks-kubeconfig.sh > aks.kubeconfig @@ -152,16 +165,14 @@ stages: fi displayName: Get Kubeconfig for AKS Cluster - # Run the E2E test suite - bash: | - az account set -s $AZURE_SUBSCRIPTION_ID - az acr login --name arosvcdev - SECRET_SA_ACCOUNT_NAME=$(SECRET_SA_ACCOUNT_NAME) make secrets . ./hack/e2e/run-rp-and-e2e.sh + az acr login --name arosvcdev + deploy_e2e_db register_sub - docker compose up --exit-code-from e2e e2e + docker compose up --quiet-pull --exit-code-from e2e e2e # Check if the E2E tests failed E2E_EXIT_CODE=$? if [ $E2E_EXIT_CODE -ne 0 ]; then diff --git a/docker-compose.yml b/docker-compose.yml index fd9833b30..a8e2ba8ce 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,3 +1,40 @@ +# These environment variables are used by ARO components. Supplying unused +# environment variables to components is harmless, so keep the reasonably common +# ones here to reduce duplication. +x-aro-common-env: + environment: &common-env + ARO_ADOPT_BY_HIVE: "true" + ARO_IMAGE: + ARO_INSTALL_VIA_HIVE: "true" + AZURE_ARM_CLIENT_ID: + AZURE_CLIENT_ID: + AZURE_CLIENT_SECRET: + AZURE_ENVIRONMENT: + AZURE_FP_CLIENT_ID: + AZURE_FP_SERVICE_PRINCIPAL_ID: + AZURE_PORTAL_ACCESS_GROUP_IDS: + AZURE_PORTAL_CLIENT_ID: + AZURE_PORTAL_ELEVATED_GROUP_IDS: + AZURE_RP_CLIENT_ID: + AZURE_RP_CLIENT_SECRET: + AZURE_SERVICE_PRINCIPAL_ID: + AZURE_SUBSCRIPTION_ID: + AZURE_TENANT_ID: + CI: + DATABASE_ACCOUNT_NAME: + DATABASE_NAME: + DOMAIN_NAME: + KEYVAULT_PREFIX: + LOCATION: + PARENT_DOMAIN_NAME: + PARENT_DOMAIN_RESOURCEGROUP: + PROXY_HOSTNAME: + PULL_SECRET: + RESOURCEGROUP: + RP_MODE: + STORAGE_ACCOUNT_DOMAIN: + USER_PULL_SECRET: + services: vpn: image: ${LOCAL_VPN_IMAGE}:${VERSION} @@ -10,7 +47,7 @@ services: volumes: - ${PWD}/secrets:/secrets:z devices: - - /dev/net/tun # required to modify VPN on host + - /dev/net/tun # required to modify VPN on host entrypoint: "openvpn" command: ["/secrets/vpn-eastus.ovpn"] ports: @@ -35,7 +72,7 @@ services: retries: 3 rp: - image: ${LOCAL_ARO_RP_IMAGE}:${VERSION} # Using localhost/aro for local image + image: ${LOCAL_ARO_RP_IMAGE}:${VERSION} # Using localhost/aro for local image build: context: . dockerfile: Dockerfile.ci-rp @@ -62,47 +99,19 @@ services: - source: hive-kubeconfig target: /app/secrets/aks.kubeconfig environment: - - ADMIN_OBJECT_ID - - ARO_IMAGE - - AZURE_ARM_CLIENT_ID - - AZURE_CLIENT_ID - - AZURE_CLIENT_SECRET - - AZURE_ENVIRONMENT - - AZURE_FP_CLIENT_ID - - AZURE_FP_SERVICE_PRINCIPAL_ID - - AZURE_GATEWAY_CLIENT_ID - - AZURE_GATEWAY_CLIENT_SECRET - - AZURE_GATEWAY_SERVICE_PRINCIPAL_ID - - AZURE_PORTAL_ACCESS_GROUP_IDS - - AZURE_PORTAL_CLIENT_ID - - AZURE_RP_CLIENT_ID - - AZURE_RP_CLIENT_SECRET - - AZURE_SERVICE_PRINCIPAL_ID - - AZURE_SUBSCRIPTION_ID - - AZURE_TENANT_ID - - DATABASE_ACCOUNT_NAME - - DATABASE_NAME - - DOMAIN_NAME - - KEYVAULT_PREFIX - - LOCATION - - MOCK_MSI_CERT - - MOCK_MSI_CLIENT_ID - - MOCK_MSI_TENANT_ID - - MOCK_MSI_OBJECT_ID - - OIDC_STORAGE_ACCOUNT_NAME - - PARENT_DOMAIN_NAME - - PARENT_DOMAIN_RESOURCEGROUP - - PROXY_HOSTNAME - - PULL_SECRET - - RESOURCEGROUP - - SECRET_SA_ACCOUNT_NAME - - STORAGE_ACCOUNT_DOMAIN - - ARO_ADOPT_BY_HIVE=true - - ARO_CHECKOUT_PATH=/app - - ARO_INSTALL_VIA_HIVE=true - - HIVE_KUBE_CONFIG_PATH=/app/secrets/aks.kubeconfig - - KUBECONFIG=/app/secrets/aks.kubeconfig - - RP_MODE=development + <<: *common-env + ADMIN_OBJECT_ID: + ARO_CHECKOUT_PATH: /app + AZURE_GATEWAY_CLIENT_ID: + AZURE_GATEWAY_CLIENT_SECRET: + AZURE_GATEWAY_SERVICE_PRINCIPAL_ID: + HIVE_KUBE_CONFIG_PATH: /app/secrets/aks.kubeconfig + KUBECONFIG: /app/secrets/aks.kubeconfig + MOCK_MSI_CERT: + MOCK_MSI_CLIENT_ID: + MOCK_MSI_OBJECT_ID: + MOCK_MSI_TENANT_ID: + OIDC_STORAGE_ACCOUNT_NAME: ports: - "8443:8443" healthcheck: @@ -120,21 +129,8 @@ services: rp: condition: service_healthy environment: - - RP_MODE - - AZURE_SUBSCRIPTION_ID - - AZURE_TENANT_ID - - LOCATION - - RESOURCEGROUP - - AZURE_PORTAL_CLIENT_ID - - AZURE_PORTAL_ELEVATED_GROUP_IDS - - AZURE_PORTAL_ACCESS_GROUP_IDS - - AZURE_RP_CLIENT_SECRET - - AZURE_RP_CLIENT_ID - - KEYVAULT_PREFIX - - AZURE_ENVIRONMENT=AzurePublicCloud - - DATABASE_ACCOUNT_NAME - - DATABASE_NAME - - NO_NPM=1 + <<: *common-env + NO_NPM: 1 ports: - "8444:8444" - "2222:2222" @@ -178,35 +174,24 @@ services: selenium: condition: service_healthy environment: - - ARO_SELENIUM_HOSTNAME - - AZURE_CLIENT_ID - - AZURE_CLIENT_SECRET - - AZURE_FP_CLIENT_ID - - AZURE_FP_SERVICE_PRINCIPAL_ID - - AZURE_PORTAL_ELEVATED_GROUP_IDS - - AZURE_RP_CLIENT_ID - - AZURE_RP_CLIENT_SECRET - - AZURE_SERVICE_PRINCIPAL_ID - - AZURE_SUBSCRIPTION_ID - - AZURE_TENANT_ID - - CI - - CLUSTER - - DATABASE_ACCOUNT_NAME - - DATABASE_NAME - - E2E_DELETE_CLUSTER - - E2E_LABEL - - KEYVAULT_PREFIX - - LOCATION - - OS_CLUSTER_VERSION - - PROXY_HOSTNAME - - PULL_SECRET - - RESOURCEGROUP - - RP_BASE_URL=https://localhost:8443 - - PORTAL_HOSTNAME=https://localhost:8444 - - RP_MODE - - USER_PULL_SECRET + <<: *common-env + ARO_SELENIUM_HOSTNAME: + CLUSTER: + E2E_DELETE_CLUSTER: + E2E_LABEL: + OS_CLUSTER_VERSION: + PORTAL_HOSTNAME: "https://localhost:8444" + RP_BASE_URL: "https://localhost:8443" entrypoint: "/usr/local/bin/e2e.test" - command: ["-test.v", "--ginkgo.v", "--ginkgo.timeout=180m", "--ginkgo.flake-attempts=2", "--ginkgo.junit-report=/tmp/e2e-report.xml", "--ginkgo.label-filter=${E2E_LABEL}"] + command: + [ + "-test.v", + "--ginkgo.v", + "--ginkgo.timeout=180m", + "--ginkgo.flake-attempts=2", + "--ginkgo.junit-report=/tmp/e2e-report.xml", + "--ginkgo.label-filter=${E2E_LABEL}", + ] secrets: proxy-client-key: diff --git a/hack/e2e/run-rp-and-e2e.sh b/hack/e2e/run-rp-and-e2e.sh index c20f67c4c..185c68657 100755 --- a/hack/e2e/run-rp-and-e2e.sh +++ b/hack/e2e/run-rp-and-e2e.sh @@ -2,6 +2,7 @@ ######## Helper file to run E2e either locally or using Azure DevOps Pipelines ######## if [[ $CI ]]; then + echo "########## CI mode enabled ##########" set -o pipefail . secrets/env @@ -198,11 +199,13 @@ echo echo "LOCATION=$LOCATION" echo "AZURE_SUBSCRIPTION_ID=$AZURE_SUBSCRIPTION_ID" echo +echo "CI=$CI" echo "RP_MODE=$RP_MODE" echo echo "DATABASE_ACCOUNT_NAME=$DATABASE_ACCOUNT_NAME" echo "DATABASE_NAME=$DATABASE_NAME" echo "RESOURCEGROUP=$RESOURCEGROUP" +echo "KEYVAULT_PREFIX=$KEYVAULT_PREFIX" echo echo "CLUSTER=$CLUSTER" echo