More sensible docker caching strategy for Prod images (#9547)

Local caching is now default strategy when building the Production image. You can still change it to pulled - similar to CI builds by providing the right build flag and this is what is used in CI by default. The flags in Breeze are now updated to be more eplanatory and friendly (build-cache-*) and a flag for "disabled" cache option is added as well. Also the Dockerfile and Dockerfile.ci files are not needed any more in the docker context. They used to be needed when we built the Kubernetes image in the container, but since we are now using production image directly - we do not need them any nmore. Combining setting the default strategy to local and removing the Dockerfile from the context has the nice effect that you can iterate much faster on the Production image without triggering rebuilds of half of the docker image as soon as the Dockerfile changes.
2020-06-28 17:38:17 +02:00 · 2020-06-28 17:38:17 +02:00 · 6aabd9a86e
--- a/.dockerignore
+++ b/.dockerignore
@ -44,8 +44,6 @@
 !pylintrc
 !pytest.ini
 !CHANGELOG.txt
-!Dockerfile.ci
-!Dockerfile
 !LICENSE
 !MANIFEST.in
 !NOTICE
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@ -64,7 +64,7 @@ jobs:
      - name: "Free space"
        run: ./scripts/ci/ci_free_space_on_ci.sh
      - name: "Build CI image"
-        run: ./scripts/ci/ci_prepare_image_on_ci.sh
+        run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
      - name: "Static checks"
        if: success()
        env:
@ -97,7 +97,7 @@ jobs:
      - name: "Free space"
        run: ./scripts/ci/ci_free_space_on_ci.sh
      - name: "Build CI image"
-        run: ./scripts/ci/ci_prepare_image_on_ci.sh
+        run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
      - name: "Static checks"
        run: |
          python -m pip install pre-commit \
@ -113,7 +113,7 @@ jobs:
    steps:
      - uses: actions/checkout@master
      - name: "Build CI image ${{ matrix.python-version }}"
-        run: ./scripts/ci/ci_prepare_image_on_ci.sh
+        run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
      - name: "Build docs"
        run: ./scripts/ci/ci_docs.sh

@ -150,7 +150,7 @@ jobs:
      - name: "Free space"
        run: ./scripts/ci/ci_free_space_on_ci.sh
      - name: "Build CI image ${{ matrix.python-version }}"
-        run: ./scripts/ci/ci_prepare_image_on_ci.sh
+        run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
      - name: "Prepare & test backport packages"
        run: |
          ./scripts/ci/ci_prepare_and_test_backport_packages.sh
@ -254,7 +254,7 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
      - name: "Free space"
        run: ./scripts/ci/ci_free_space_on_ci.sh
      - name: "Build CI image ${{ matrix.python-version }}"
-        run: ./scripts/ci/ci_prepare_image_on_ci.sh
+        run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
      - name: "Tests"
        run: ./scripts/ci/ci_run_airflow_testing.sh

@ -286,7 +286,7 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
      - name: "Free space"
        run: ./scripts/ci/ci_free_space_on_ci.sh
      - name: "Build CI image ${{ matrix.python-version }}"
-        run: ./scripts/ci/ci_prepare_image_on_ci.sh
+        run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
      - name: "Tests"
        run: ./scripts/ci/ci_run_airflow_testing.sh

@ -316,7 +316,7 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
      - name: "Free space"
        run: ./scripts/ci/ci_free_space_on_ci.sh
      - name: "Build CI image ${{ matrix.python-version }}"
-        run: ./scripts/ci/ci_prepare_image_on_ci.sh
+        run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
      - name: "Tests"
        run: ./scripts/ci/ci_run_airflow_testing.sh

@ -349,7 +349,7 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
      - name: "Free space"
        run: ./scripts/ci/ci_free_space_on_ci.sh
      - name: "Build CI image ${{ matrix.python-version }}"
-        run: ./scripts/ci/ci_prepare_image_on_ci.sh
+        run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
      - name: "Tests"
        run: ./scripts/ci/ci_run_airflow_testing.sh

@ -372,7 +372,7 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
      - name: "Free space"
        run: ./scripts/ci/ci_free_space_on_ci.sh
      - name: "Build CI image ${{ matrix.python-version }}"
-        run: ./scripts/ci/ci_prepare_image_on_ci.sh
+        run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
      - name: "Generate requirements"
        run: ./scripts/ci/ci_generate_requirements.sh

@ -432,7 +432,7 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
      - name: "Free space"
        run: ./scripts/ci/ci_free_space_on_ci.sh
      - name: "Build CI image"
-        run: ./scripts/ci/ci_prepare_image_on_ci.sh
+        run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
      - name: "Push CI image ${{ matrix.python-version }}"
        run: ./scripts/ci/ci_push_ci_image.sh

--- a/BREEZE.rst
+++ b/BREEZE.rst
@ -792,7 +792,9 @@ This is the current syntax for  `./breeze <./breeze>`_:

        Builds docker image (CI or production) without entering the container. You can pass
        additional options to this command, such as '--force-build-image',
-        '--force-pull-image' '--python' '--use-local-cache'' in order to modify build behaviour.
+        '--force-pull-image', '--python', '--build-cache-local' or '-build-cache-pulled'
+        in order to modify build behaviour.
+
        You can also pass '--production-image' flag to build production image rather than CI image.

  Flags:
@ -856,9 +858,32 @@ This is the current syntax for  `./breeze <./breeze>`_:
          Force build images with cache disabled. This will remove the pulled or build images
          and start building images from scratch. This might take a long time.

-  -L, --use-local-cache
+  -L, --build-cache-local
          Uses local cache to build images. No pulled images will be used, but results of local
-          builds in the Docker cache are used instead.
+          builds in the Docker cache are used instead. This will take longer than when the pulled
+          cache is used for the first time, but subsequent '--build-cache-local' builds will be
+          faster as they will use mostly the locally build cache.
+
+          This is default strategy used by the Production image builds.
+
+  -U, --build-cache-pulled
+          Uses images pulled from registry (either DockerHub or GitHub depending on
+          --github-registry flag) to build images. The pulled images will be used as cache.
+          Those builds are usually faster than when ''--build-cache-local'' with the exception if
+          the registry images are not yet updated. The DockerHub images are updated nightly and the
+          GitHub images are updated after merges to master so it might be that the images are still
+          outdated vs. the latest version of the Dockerfiles you are using. In this case, the
+          ''--build-cache-local'' might be faster, especially if you iterate and change the
+          Dockerfiles yourself.
+
+          This is default strategy used by the CI image builds.
+
+  -X, --build-cache-disabled
+          Disables cache during docker builds. This is useful if you want to make sure you want to
+          rebuild everything from scratch.
+
+          This strategy is used by default for both Production and CI images for the scheduled
+          (nightly) builds in CI.

  -D, --dockerhub-user
          DockerHub user used to pull, push and build images. Default: apache.
@ -1219,9 +1244,32 @@ This is the current syntax for  `./breeze <./breeze>`_:
          Force build images with cache disabled. This will remove the pulled or build images
          and start building images from scratch. This might take a long time.

-  -L, --use-local-cache
+  -L, --build-cache-local
          Uses local cache to build images. No pulled images will be used, but results of local
-          builds in the Docker cache are used instead.
+          builds in the Docker cache are used instead. This will take longer than when the pulled
+          cache is used for the first time, but subsequent '--build-cache-local' builds will be
+          faster as they will use mostly the locally build cache.
+
+          This is default strategy used by the Production image builds.
+
+  -U, --build-cache-pulled
+          Uses images pulled from registry (either DockerHub or GitHub depending on
+          --github-registry flag) to build images. The pulled images will be used as cache.
+          Those builds are usually faster than when ''--build-cache-local'' with the exception if
+          the registry images are not yet updated. The DockerHub images are updated nightly and the
+          GitHub images are updated after merges to master so it might be that the images are still
+          outdated vs. the latest version of the Dockerfiles you are using. In this case, the
+          ''--build-cache-local'' might be faster, especially if you iterate and change the
+          Dockerfiles yourself.
+
+          This is default strategy used by the CI image builds.
+
+  -X, --build-cache-disabled
+          Disables cache during docker builds. This is useful if you want to make sure you want to
+          rebuild everything from scratch.
+
+          This strategy is used by default for both Production and CI images for the scheduled
+          (nightly) builds in CI.


  ####################################################################################################
@ -1561,9 +1609,32 @@ This is the current syntax for  `./breeze <./breeze>`_:
          Force build images with cache disabled. This will remove the pulled or build images
          and start building images from scratch. This might take a long time.

-  -L, --use-local-cache
+  -L, --build-cache-local
          Uses local cache to build images. No pulled images will be used, but results of local
-          builds in the Docker cache are used instead.
+          builds in the Docker cache are used instead. This will take longer than when the pulled
+          cache is used for the first time, but subsequent '--build-cache-local' builds will be
+          faster as they will use mostly the locally build cache.
+
+          This is default strategy used by the Production image builds.
+
+  -U, --build-cache-pulled
+          Uses images pulled from registry (either DockerHub or GitHub depending on
+          --github-registry flag) to build images. The pulled images will be used as cache.
+          Those builds are usually faster than when ''--build-cache-local'' with the exception if
+          the registry images are not yet updated. The DockerHub images are updated nightly and the
+          GitHub images are updated after merges to master so it might be that the images are still
+          outdated vs. the latest version of the Dockerfiles you are using. In this case, the
+          ''--build-cache-local'' might be faster, especially if you iterate and change the
+          Dockerfiles yourself.
+
+          This is default strategy used by the CI image builds.
+
+  -X, --build-cache-disabled
+          Disables cache during docker builds. This is useful if you want to make sure you want to
+          rebuild everything from scratch.
+
+          This strategy is used by default for both Production and CI images for the scheduled
+          (nightly) builds in CI.

  ****************************************************************************************************
   Flags for pulling/pushing Docker images (both CI and production)
--- a/IMAGES.rst
+++ b/IMAGES.rst
@ -115,36 +115,56 @@ parameter to Breeze:
 Using cache during builds
 =========================

-Default mechanism used in Breeze for building images uses - as base - images puled from DockerHub or
-GitHub Image Registry. This is in order to speed up local builds and CI builds - instead of 15 minutes
+Default mechanism used in Breeze for building CI images uses images pulled from DockerHub or
+GitHub Image Registry. This is done to speed up local builds and CI builds - instead of 15 minutes
 for rebuild of CI images, it takes usually less than 3 minutes when cache is used. For CI builds this is
-usually the best strategy - to use default "pull" cache - same for Production Image - it's better to rely
-on the "pull" mechanism rather than rebuild the image from the scratch.
+usually the best strategy - to use default "pull" cache. This is default strategy when
+`<BREEZE.rst>`_ builds are performed.

-However when you are iterating on the images and want to rebuild them quickly and often you can provide the
-``--use-local-cache`` flag to build commands - this way the standard docker mechanism based on local cache
-will be used. The first time you run it, it will take considerably longer time than if you use the
-default pull mechanism, but then when you do small, incremental changes to local sources, Dockerfile image
-and scripts further rebuilds with --use-local-cache will be considerably faster.
+For Production Image - which is far smaller and faster to build, it's better to use local build cache (the
+standard mechanism that docker uses. This is the default strategy for production images when
+`<BREEZE.rst>`_ builds are performed. The first time you run it, it will take considerably longer time than
+if you use the pull mechanism, but then when you do small, incremental changes to local sources,
+Dockerfile image= and scripts further rebuilds with local build cache will be considerably faster.
+
+You can also disable build cache altogether. This is the strategy used by the scheduled builds in CI - they
+will always rebuild all the images from scratch.
+
+You can change the strategy by providing one of the ``--build-cache-local``, ``--build-cache-pulled`` or
+even ``--build-cache-disabled`` flags when you run Breeze commands. For example:

 .. code-block:: bash

-  ./breeze build-image --python 3.7 --production-image --use-local-cache
+  ./breeze build-image --python 3.7 --build-cache-local

-You can also turn local docker caching by setting DOCKER_CACHE variable to "local" instead of the default
-"pulled" and export it to Breeze.
+Will build the CI image using local build cache (note that it will take quite a long time the first
+time you run it).
+
+.. code-block:: bash
+
+  ./breeze build-image --python 3.7 --production-image --build-cache-pulled
+
+Will build the production image with pulled images as cache.
+
+
+.. code-block:: bash
+
+  ./breeze build-image --python 3.7 --production-image --build-cache-disabled
+
+Will build the production image from the scratch.
+
+You can also turn local docker caching by setting ``DOCKER_CACHE`` variable to "local", "pulled",
+"disabled" and exporting it.

 .. code-block:: bash

  export DOCKER_CACHE="local"

-You can also - if you really want - disable caching altogether by setting this variable to "no-cache".
-This is how "scheduled" builds in our CI are run - those builds take a long time because they
-always rebuild everything from scratch.
+or

 .. code-block:: bash

-  export DOCKER_CACHE="no-cache"
+  export DOCKER_CACHE="disabled"


 Choosing image registry
--- a/48
+++ b/48
@ -376,7 +376,7 @@ EOF
        if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
            cat <<EOF

-   Use production image.
+   Production image.

   Branch name:             ${BRANCH_NAME}
   Docker image:            ${AIRFLOW_PROD_IMAGE}
@ -384,7 +384,7 @@ EOF
        else
            cat <<EOF

-   Use CI image.
+   CI image.

   Branch name:             ${BRANCH_NAME}
   Docker image:            ${AIRFLOW_CI_IMAGE}
@ -674,11 +674,21 @@ function parse_arguments() {
          export DOCKER_CACHE="no-cache"
          export FORCE_BUILD_IMAGES="true"
          shift ;;
-        -L|--use-local-cache)
+        -L|--build-cache-local)
          echo "Use local cache to build images"
          echo
          export DOCKER_CACHE="local"
          shift ;;
+        -U|--build-cache-pulled)
+          echo "Use pulled cache to build images"
+          echo
+          export DOCKER_CACHE="pulled"
+          shift ;;
+        -X|--build-cache-disabled)
+          echo "Use disabled cache to build images"
+          echo
+          export DOCKER_CACHE="disabled"
+          shift ;;
        -P|--force-pull-images)
          echo "Force pulling images before build. Uses pulled images as cache."
          echo
@ -1083,7 +1093,9 @@ ${CMDNAME} build-image [FLAGS]

      Builds docker image (CI or production) without entering the container. You can pass
      additional options to this command, such as '--force-build-image',
-      '--force-pull-image' '--python' '--use-local-cache'' in order to modify build behaviour.
+      '--force-pull-image', '--python', '--build-cache-local' or '-build-cache-pulled'
+      in order to modify build behaviour.
+
      You can also pass '--production-image' flag to build production image rather than CI image.

 Flags:
@ -1614,9 +1626,33 @@ ${FORMATTED_DEFAULT_PROD_EXTRAS}
        Force build images with cache disabled. This will remove the pulled or build images
        and start building images from scratch. This might take a long time.

-L, --use-local-cache
+-L, --build-cache-local
        Uses local cache to build images. No pulled images will be used, but results of local
-        builds in the Docker cache are used instead.
+        builds in the Docker cache are used instead. This will take longer than when the pulled
+        cache is used for the first time, but subsequent '--build-cache-local' builds will be
+        faster as they will use mostly the locally build cache.
+
+        This is default strategy used by the Production image builds.
+
+-U, --build-cache-pulled
+        Uses images pulled from registry (either DockerHub or GitHub depending on
+        --github-registry flag) to build images. The pulled images will be used as cache.
+        Those builds are usually faster than when ''--build-cache-local'' with the exception if
+        the registry images are not yet updated. The DockerHub images are updated nightly and the
+        GitHub images are updated after merges to master so it might be that the images are still
+        outdated vs. the latest version of the Dockerfiles you are using. In this case, the
+        ''--build-cache-local'' might be faster, especially if you iterate and change the
+        Dockerfiles yourself.
+
+        This is default strategy used by the CI image builds.
+
+-X, --build-cache-disabled
+        Disables cache during docker builds. This is useful if you want to make sure you want to
+        rebuild everything from scratch.
+
+        This strategy is used by default for both Production and CI images for the scheduled
+        (nightly) builds in CI.
+
 "
 }

--- a/6
+++ b/6
@ -103,7 +103,8 @@ h p: b: i:
 K: V:
 l a: t: d:
 v y n q f
-F P I E: C L
+F P I E: C
+L U X
 D: R: c g: G:
 S: N:
 "
@ -113,7 +114,8 @@ help python: backend: integration:
 kubernetes-mode: kubernetes-version:
 skip-mounting-local-sources install-airflow-version: install-airflow-reference: db-reset
 verbose assume-yes assume-no assume-quit forward-credentials
-force-build-images force-pull-images production-image extras: force-clean-images use-local-cache
+force-build-images force-pull-images production-image extras: force-clean-images
+build-cache-local build-cache-pulled build-cache-disabled
 dockerhub-user: dockerhub-repo: github-registry github-organisation: github-repo:
 postgres-version: mysql-version:
 version-suffix-for-pypi: version-suffix-for-svn:
--- a/scripts/ci/ci_prepare_ci_image_on_ci.sh
+++ b/scripts/ci/ci_prepare_ci_image_on_ci.sh
@ -18,7 +18,4 @@
 # shellcheck source=scripts/ci/_script_init.sh
 . "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"

-export UPGRADE_TO_LATEST_REQUIREMENTS="false"
-export SKIP_CI_IMAGE_CHECK="false"
-
 build_ci_image_on_ci
--- a/scripts/ci/ci_prepare_prod_image_on_ci.sh
+++ b/scripts/ci/ci_prepare_prod_image_on_ci.sh
@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# shellcheck source=scripts/ci/_script_init.sh
+. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
+
+build_prod_image_on_ci
--- a/scripts/ci/libraries/_build_images.sh
+++ b/scripts/ci/libraries/_build_images.sh
@ -316,6 +316,11 @@ function print_build_info() {
 # Prepares all variables needed by the CI build. Depending on the configuration used (python version
 # DockerHub user etc. the variables are set so that other functions can use those variables.
 function prepare_ci_build() {
+    # We use pulled docker image cache by default for CI images to  speed up the builds
+    export DOCKER_CACHE=${DOCKER_CACHE:="pulled"}
+    echo
+    echo "Using ${DOCKER_CACHE} cache strategy for the build."
+    echo
    export AIRFLOW_CI_BASE_TAG="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-ci"
    export AIRFLOW_CI_LOCAL_MANIFEST_IMAGE="local/${DOCKERHUB_REPO}:${AIRFLOW_CI_BASE_TAG}-manifest"
    export AIRFLOW_CI_REMOTE_MANIFEST_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_CI_BASE_TAG}-manifest"
@ -473,25 +478,39 @@ function rebuild_ci_image_if_needed_and_confirmed() {
    fi
 }

+# Determines the strategy to be used for caching based on the type of CI job run.
+# In case of CRON jobs we run builds without cache and upgrade to latest requirements
+function determine_cache_strategy() {
+    if [[ "${CI_EVENT_TYPE:=}" == "schedule" ]]; then
+        echo
+        echo "Disabling cache for scheduled jobs"
+        echo
+        export DOCKER_CACHE="disabled"
+        echo
+        echo "Requirements are upgraded to latest for scheduled CI build"
+        echo
+        export UPGRADE_TO_LATEST_REQUIREMENTS="true"
+    else
+        echo
+        echo "Pull cache used for regular CI builds"
+        echo
+        export DOCKER_CACHE="pulled"
+        echo
+        echo "Requirements are not upgraded to latest ones for regular CI builds"
+        echo
+        export UPGRADE_TO_LATEST_REQUIREMENTS="false"
+    fi
+}
+

 # Builds the CI image in the CI environment.
 # Depending on the type of build (push/pr/scheduled) it will either build it incrementally or
 # from the scratch without cache (the latter for scheduled builds only)
 function build_ci_image_on_ci() {
+    export SKIP_CI_IMAGE_CHECK="false"
+
    get_ci_environment
-
-    # In case of CRON jobs we run builds without cache and upgrade to latest requirements
-    if [[ "${CI_EVENT_TYPE:=}" == "schedule" ]]; then
-        echo
-        echo "Disabling cache for scheduled jobs"
-        echo
-        export DOCKER_CACHE="no-cache"
-        echo
-        echo "Requirements are upgraded to latest while running Docker build"
-        echo
-        export UPGRADE_TO_LATEST_REQUIREMENTS="true"
-    fi
-
+    determine_cache_strategy
    prepare_ci_build

    rm -rf "${BUILD_CACHE_DIR}"
@ -500,12 +519,14 @@ function build_ci_image_on_ci() {
    rebuild_ci_image_if_needed

    # Disable force pulling forced above this is needed for the subsequent scripts so that
-    # They do not try to pull/build images again
+    # They do not try to pull/build images again. Also skip the image check entirely for
+    # the rest of the script
    unset FORCE_PULL_IMAGES
    unset FORCE_BUILD
+    export SKIP_CI_IMAGE_CHECK="true"
 }

-# Builds CI image - depending on the caching strategy (pulled, local, no-cache) it
+# Builds CI image - depending on the caching strategy (pulled, local, disabled) it
 # passes the necessary docker build flags via DOCKER_CACHE_CI_DIRECTIVE array
 # it also passes the right Build args depending on the configuration of the build
 # selected by Breeze flags or environment variables.
@ -521,7 +542,7 @@ function build_ci_image() {
    fi
    pull_ci_image_if_needed

-    if [[ "${DOCKER_CACHE}" == "no-cache" ]]; then
+    if [[ "${DOCKER_CACHE}" == "disabled" ]]; then
        export DOCKER_CACHE_CI_DIRECTIVE=("--no-cache")
    elif [[ "${DOCKER_CACHE}" == "local" ]]; then
        export DOCKER_CACHE_CI_DIRECTIVE=()
@ -570,6 +591,11 @@ Docker building ${AIRFLOW_CI_IMAGE}.
 # Prepares all variables needed by the CI build. Depending on the configuration used (python version
 # DockerHub user etc. the variables are set so that other functions can use those variables.
 function prepare_prod_build() {
+    # We use local docker image cache by default for Production images
+    export DOCKER_CACHE=${DOCKER_CACHE:="local"}
+    echo
+    echo "Using ${DOCKER_CACHE} cache strategy for the build."
+    echo
    if [[ "${INSTALL_AIRFLOW_REFERENCE:=}" != "" ]]; then
        # When --install-airflow-reference is used then the image is build from github tag
        EXTRA_DOCKER_PROD_BUILD_FLAGS=(
@ -638,7 +664,29 @@ function prepare_prod_build() {
 }


-# Builds PROD image - depending on the caching strategy (pulled, local, no-cache) it
+# Builds the prod image in the CI environment.
+# Depending on the type of build (push/pr/scheduled) it will either build it incrementally or
+# from the scratch without cache (the latter for scheduled builds only)
+function build_prod_image_on_ci() {
+    get_prod_environment
+
+    determine_cache_strategy
+
+    prepare_prod_build
+
+    rm -rf "${BUILD_CACHE_DIR}"
+    mkdir -pv "${BUILD_CACHE_DIR}"
+
+    build_prod_image
+
+    # Disable force pulling forced above this is needed for the subsequent scripts so that
+    # They do not try to pull/build images again
+    unset FORCE_PULL_IMAGES
+    unset FORCE_BUILD
+}
+
+
+# Builds PROD image - depending on the caching strategy (pulled, local, disabled) it
 # passes the necessary docker build flags via DOCKER_CACHE_PROD_DIRECTIVE and
 # DOCKER_CACHE_PROD_BUILD_DIRECTIVE (separate caching options are needed for "build" segment of the image)
 # it also passes the right Build args depending on the configuration of the build
@ -647,7 +695,7 @@ function build_prod_image() {
    print_build_info
    pull_prod_images_if_needed

-    if [[ "${DOCKER_CACHE}" == "no-cache" ]]; then
+    if [[ "${DOCKER_CACHE}" == "disabled" ]]; then
        export DOCKER_CACHE_PROD_DIRECTIVE=("--cache-from" "${AIRFLOW_PROD_BUILD_IMAGE}")
        export DOCKER_CACHE_PROD_BUILD_DIRECTIVE=("--no-cache")
    elif [[ "${DOCKER_CACHE}" == "local" ]]; then
--- a/scripts/ci/libraries/_initialization.sh
+++ b/scripts/ci/libraries/_initialization.sh
@ -161,11 +161,6 @@ function initialize_common_environment {
        done
    fi

-    # We use pulled docker image cache by default to speed up the builds
-    # but we can also set different docker caching strategy (for example we can use local cache
-    # to build the images in case we iterate with the image
-    export DOCKER_CACHE=${DOCKER_CACHE:="pulled"}
-
    # By default we are not upgrading to latest requirements when building Docker CI image
    # This will only be done in cron jobs
    export UPGRADE_TO_LATEST_REQUIREMENTS=${UPGRADE_TO_LATEST_REQUIREMENTS:="false"}