Merge pull request #3875 from Azure/hawkowl/e2e-master-smoke-test

Don't run long-running regression tests on PRs

.pipelines/e2e.yml

@@ -51,6 +51,13 @@ jobs:
           make extract-aro-docker
         displayName: Extract ARO binaries from build
 
+      # Override the E2E label for IndividualCI/BatchedCI (i.e. not manually
+      # ran/PR jobs) to run all non-smoke tasks (default is !smoke&&!regressiontest)
+      - script: |
+          echo "##vso[task.setvariable variable=E2E_LABEL]!smoke"
+        displayName: Enable regression tests in CI
+        condition: in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')
+
       - script: |
           az account set -s $AZURE_SUBSCRIPTION_ID
           SECRET_SA_ACCOUNT_NAME=$(SECRET_SA_ACCOUNT_NAME) make secrets

Makefile

@@ -3,7 +3,7 @@ TAG ?= $(shell git describe --exact-match 2>/dev/null)
 COMMIT = $(shell git rev-parse --short=7 HEAD)$(shell [[ $$(git status --porcelain) = "" ]] || echo -dirty)
 ARO_IMAGE_BASE = ${RP_IMAGE_ACR}.azurecr.io/aro
 E2E_FLAGS ?= -test.v --ginkgo.v --ginkgo.timeout 180m --ginkgo.flake-attempts=2 --ginkgo.junit-report=e2e-report.xml
-E2E_LABEL ?= !smoke
+E2E_LABEL ?= !smoke&&!regressiontest
 GO_FLAGS ?= -tags=containers_image_openpgp,exclude_graphdriver_btrfs,exclude_graphdriver_devicemapper
 
 export GOFLAGS=$(GO_FLAGS)

test/e2e/adminapi_redeployvm.go

@@ -27,7 +27,7 @@ const (
 	uptimeStrFmt = "2006-01-02 15:04:05" // https://go.dev/src/time/format.go
 )
 
-var _ = Describe("[Admin API] VM redeploy action", func() {
+var _ = Describe("[Admin API] VM redeploy action", Label(regressiontest), func() {
 	BeforeEach(skipIfNotInDevelopmentEnv)
 
 	It("must trigger a selected VM to redeploy", func(ctx context.Context) {

test/e2e/dns.go

@@ -40,7 +40,7 @@ const (
 	resolvConfContainerName = "read-resolv-conf"
 )
 
-var _ = Describe("ARO cluster DNS", func() {
+var _ = Describe("ARO cluster DNS", Label(regressiontest), func() {
 	BeforeEach(skipIfNotInDevelopmentEnv)
 
 	It("must not be adversely affected by Azure host servicing", func(ctx context.Context) {

test/e2e/setup.go

@@ -62,6 +62,10 @@ import (
 
 const (
 	smoke = "smoke"
+	// regressiontest is for tests designed to ensure that something doesn't
+	// break before we go to release, but doesn't need to be validated in every
+	// PR.
+	regressiontest = "regressiontest"
 )
 
 //go:embed static_resources