Add a possibility to switch back to building images by secret (#10509)

You can now define secret in your own fork:

AIRFLOW_GITHUB_REGISTRY_WAIT_FOR_IMAGE

If you set it to "false", it skips building images in separate
workflow_run - images will be built in the jobs run in the
CI Build run and they won't be pushed to the registry.

Note - you can't have secrets starting with GITHUB_, that's why
the AIRFLOW_* prefix

.github/workflows/build-images-workflow-run.yml

@@ -39,6 +39,7 @@ env:
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
   GITHUB_REGISTRY_PULL_IMAGE_TAG: "latest"
   GITHUB_REGISTRY_WAIT_FOR_IMAGE: "false"
+  BUILD_IMAGES: ${{ secrets.AIRFLOW_GITHUB_REGISTRY_WAIT_FOR_IMAGE != 'false' }}
 
 jobs:
 
@@ -52,6 +53,7 @@ jobs:
       sourceHeadSha: ${{ steps.cancel.outputs.sourceHeadSha }}
       sourceEvent: ${{ steps.cancel.outputs.sourceEvent }}
       cacheDirective: ${{ steps.cache-directive.outputs.docker-cache }}
+      buildImages: ${{ steps.build-images.outputs.buildImages }}
     if: github.repository == 'apache/airflow' || github.event.workflow_run.event != 'schedule'
     steps:
       - name: "Cancel duplicated 'CI Build' runs"
@@ -61,6 +63,12 @@ jobs:
           token: ${{ secrets.GITHUB_TOKEN }}
           cancelMode: duplicates
           sourceRunId: ${{ github.event.workflow_run.id }}
+      - name: "Output BUILD_IMAGES"
+        id: build-images
+        run: |
+          # Workaround - jobs cannot access env variable in "ifs"
+          # https://github.community/t/how-to-set-and-access-a-workflow-variable/17335/16
+          echo "::set-output name=buildImages::${BUILD_IMAGES}"
       - name: "Cancel duplicated 'Build Image' runs"
 
         # We find duplicates of our own "Build Image" runs - due to a missing feature
@@ -75,6 +83,7 @@ jobs:
             [".*Event: ${{ steps.cancel.outputs.sourceEvent }}
             Repo: ${{ steps.cancel.outputs.sourceHeadRepo }}
             Branch: ${{ steps.cancel.outputs.sourceHeadBranch }}.*"]
+        if: env.BUILD_IMAGES == 'true'
       - name: "Cancel all 'CI Build' runs where some jobs failed"
 
         # We find any of the "CI Build" workflow runs, where any of the important jobs
@@ -118,7 +127,7 @@ jobs:
         # it to cancel any jobs that have matching names containing Source Run Id:
         # followed by one of the run ids. Yes I know it's super complex ¯\_(ツ)_/¯.
 
-        if: steps.cancel-failed.outputs.cancelledRuns != '[]'
+        if: env.BUILD_IMAGES == 'true' && steps.cancel-failed.outputs.cancelledRuns != '[]'
         uses: potiuk/cancel-workflow-runs@v2
         with:
           cancelMode: namedJobs
@@ -149,9 +158,11 @@ jobs:
       Source Sha: ${{ needs.cancel-workflow-runs.outputs.sourceHeadSha }}
     runs-on: ubuntu-latest
     needs: [cancel-workflow-runs]
-    if: github.repository == 'apache/airflow' || github.event.workflow_run.event != 'schedule'
     env:
       GITHUB_CONTEXT: ${{ toJson(github) }}
+    if: >
+      needs.cancel-workflow-runs.outputs.buildImages == 'true' &&
+      (github.repository == 'apache/airflow' || github.event.workflow_run.event != 'schedule')
     steps:
       - name: >
           Event: ${{ needs.cancel-workflow-runs.outputs.sourceEvent }}
@@ -174,7 +185,9 @@ jobs:
         python-version: [3.6, 3.7, 3.8]
         image-type: [CI, PROD]
       fail-fast: true
-    if: github.repository == 'apache/airflow' || github.event.workflow_run.event != 'schedule'
+    if: >
+      needs.cancel-workflow-runs.outputs.buildImages == 'true' &&
+      (github.repository == 'apache/airflow' || github.event.workflow_run.event != 'schedule')
     env:
       BACKEND: postgres
       PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}

.github/workflows/ci.yml

@@ -56,9 +56,10 @@ env:
   #
   #  GITHUB_REGISTRY_WAIT_FOR_IMAGE: "false"
   #
+  # You can also switch back to building images locally and disabling the "Build Images" workflow
+  # by defining AIRFLOW_GITHUB_REGISTRY_WAIT_FOR_IMAGE secret with value set to "false"
 
-  GITHUB_REGISTRY_WAIT_FOR_IMAGE: "true"
-
+  GITHUB_REGISTRY_WAIT_FOR_IMAGE: ${{ secrets.AIRFLOW_GITHUB_REGISTRY_WAIT_FOR_IMAGE != 'false' }}
 
 jobs:
 
@@ -305,7 +306,7 @@ jobs:
       POSTGRES_VERSION: ${{ matrix.postgres-version }}
       RUN_TESTS: true
       TEST_TYPE: ${{ matrix.test-type }}
-    if: |
+    if: >
         (needs.trigger-tests.outputs.run-tests == 'true' || github.event_name != 'pull_request') &&
         (github.repository == 'apache/airflow' || github.event_name != 'schedule')
     steps:
@@ -351,7 +352,7 @@ jobs:
       MYSQL_VERSION: ${{ matrix.mysql-version }}
       RUN_TESTS: true
       TEST_TYPE: ${{ matrix.test-type }}
-    if: |
+    if: >
         (needs.trigger-tests.outputs.run-tests == 'true' || github.event_name != 'pull_request') &&
         (github.repository == 'apache/airflow' || github.event_name != 'schedule')
     steps:
@@ -440,7 +441,7 @@ jobs:
       TEST_TYPE: Quarantined
       NUM_RUNS: 20
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-    if: |
+    if: >
       (needs.trigger-tests.outputs.run-tests == 'true' || github.event_name != 'pull_request') &&
       (github.repository == 'apache/airflow' || github.event_name != 'schedule')
     steps:
@@ -561,7 +562,7 @@ jobs:
       KUBERNETES_VERSION: "${{ matrix.kubernetes-version }}"
       KIND_VERSION: "${{ matrix.kind-version }}"
       HELM_VERSION: "${{ matrix.helm-version }}"
-    if: |
+    if: >
       (needs.trigger-tests.outputs.run-tests == 'true' || github.event_name != 'pull_request') &&
       (github.repository == 'apache/airflow' || github.event_name != 'schedule')
     steps:
@@ -622,7 +623,7 @@ jobs:
       - prod-images
       - docs
       - docs-spell-check
-    if: |
+    if: >
       (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/v1-10-test') &&
       github.event_name != 'schedule'
     strategy:
@@ -660,7 +661,7 @@ jobs:
       - ci-images
       - docs
       - docs-spell-check
-    if: |
+    if: >
       (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/v1-10-test' ) &&
       github.event_name != 'schedule'
     strategy:
@@ -700,7 +701,7 @@ jobs:
       - tests-kubernetes
     env:
       PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
-    if: |
+    if: >
       (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/v1-10-test' ) &&
       github.event_name != 'pull' &&
       (github.repository == 'apache/airflow' || github.event_name != 'schedule')
@@ -728,7 +729,7 @@ jobs:
     name: "Constraints push"
     runs-on: ubuntu-latest
     needs: [constraints]
-    if: |
+    if: >
       (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/v1-10-test' ) &&
       github.event_name != 'pull' &&
       (github.repository == 'apache/airflow' || github.event_name != 'schedule')

.github/workflows/scheduled_quarantined.yml

@@ -77,7 +77,7 @@ jobs:
       TEST_TYPE: Quarantined
       NUM_RUNS: 20
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-    if: |
+    if: >
       github.repository == 'apache/airflow' &&
       (needs.trigger-tests.outputs.run-tests == 'true' || github.event_name != 'pull_request')
     steps: