Adds resource check when running Breeze (#14908)

When Breeze is run, it requires some resources in the docker
engine, otherwise it will produce strange errors.

This PR adds resource check when running breeze - it will print
human-friendly size of CPU/Memory/Disk available for docker
engine and red error (still allowing Docker to run) when the
resources are not enough.

Fixes: #14899
(cherry picked from commit 3dd42a5a3f)

BREEZE.rst

@@ -157,10 +157,29 @@ If you use zsh, run this command and re-login:
     echo 'export PATH="/usr/local/opt/gnu-getopt/bin:$PATH"' >> ~/.zprofile
     . ~/.zprofile
 
+
+Let's confirm that ``getopt`` and ``gstat`` utilities are successfully installed
+
+.. code-block:: bash
+
+    $ getopt --version
+    getopt from util-linux *
+    $ gstat --version
+    stat (GNU coreutils) *
+    Copyright (C) 2020 Free Software Foundation, Inc.
+    License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>.
+    This is free software: you are free to change and redistribute it.
+    There is NO WARRANTY, to the extent permitted by law.
+
+    Written by Michael Meskes.
+
+Resources required
+==================
+
 Memory
 ------
 
-Minimum 4GB RAM is required to run the full Breeze environment.
+Minimum 4GB RAM for Docker Engine is required to run the full Breeze environment.
 
 On macOS, 2GB of RAM are available for your Docker containers by default, but more memory is recommended
 (4GB should be comfortable). For details see
@@ -168,6 +187,18 @@ On macOS, 2GB of RAM are available for your Docker containers by default, but mo
 
 On Windows WSL 2 expect the Linux Distro and Docker containers to use 7 - 8 GB of RAM.
 
+Disk
+----
+
+Minimum 40GB free disk space is required for your Docker Containers.
+
+On Mac OS This might deteriorate over time so you might need to increase it or run ``docker system --prune``
+periodically. For details see
+`Docker for Mac - Advanced tab <https://docs.docker.com/v17.12/docker-for-mac/#advanced-tab>`_.
+
+On WSL2 you might want to increase your Virtual Hard Disk by following:
+`Expanding the size of your WSL 2 Virtual Hard Disk <https://docs.microsoft.com/en-us/windows/wsl/compare-versions#expanding-the-size-of-your-wsl-2-virtual-hard-disk>`_
+
 Cleaning the environment
 ------------------------
 
@@ -177,7 +208,7 @@ them, you may end up with some unused image data.
 
 To clean up the Docker environment:
 
-1. Stop Breeze with ``./breeze stop``.
+1. Stop Breeze with ``./breeze stop``. (If Breeze is already running)
 
 2. Run the ``docker system prune`` command.
 
@@ -333,7 +364,7 @@ Managing CI environment:
     * Generate constraints with ``breeze generate-constraints``
     * Execute arbitrary command in the test environment with ``breeze shell`` command
     * Execute arbitrary docker-compose command with ``breeze docker-compose`` command
-    * Push docker images with ``breeze push-image`` command (require committer's rights to push images)
+    * Push docker images with ``breeze push-image`` command (require committers rights to push images)
 
 You can optionally reset the Airflow metada database if specified as extra ``--db-reset`` flag and for CI image
 you can also start integrations (separate Docker images) if specified as extra ``--integration`` flags. You can also
@@ -356,7 +387,7 @@ Managing Prod environment (with ``--production-image`` flag):
     * Restart running interactive environment with ``breeze restart`` command
     * Execute arbitrary command in the test environment with ``breeze shell`` command
     * Execute arbitrary docker-compose command with ``breeze docker-compose`` command
-    * Push docker images with ``breeze push-image`` command (require committer's rights to push images)
+    * Push docker images with ``breeze push-image`` command (require committers rights to push images)
 
 You can optionally reset database if specified as extra ``--db-reset`` flag. You can also
 chose which backend database should be used with ``--backend`` flag and python version with ``--python`` flag.

breeze

@@ -3430,8 +3430,10 @@ function breeze::run_breeze_command() {
     else
         dc_run_file="${BUILD_CACHE_DIR}/${DOCKER_COMPOSE_RUN_SCRIPT_FOR_CI}"
     fi
+
     case "${command_to_run}" in
     enter_breeze)
+        docker_engine::check_all_resources
         if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
             ${run_command} "${dc_run_file}" run --service-ports --rm airflow "${@}"
             ${run_command} "${SCRIPTS_CI_DIR}/tools/ci_fix_ownership.sh"
@@ -3440,6 +3442,7 @@ function breeze::run_breeze_command() {
         fi
         ;;
     run_exec)
+        docker_engine::check_all_resources
         # Unfortunately `docker-compose exec` does not support exec'ing into containers started with run :(
         # so we have to find it manually
         set +e
@@ -3451,16 +3454,19 @@ function breeze::run_breeze_command() {
             "/opt/airflow/scripts/in_container/entrypoint_exec.sh" "${@}"
         ;;
     run_tests)
+        docker_engine::check_all_resources
         export RUN_TESTS="true"
         readonly RUN_TESTS
         ${run_command} "${BUILD_CACHE_DIR}/${DOCKER_COMPOSE_RUN_SCRIPT_FOR_CI}" run --service-ports --rm airflow "$@"
         ;;
     run_docker_compose)
+        docker_engine::check_all_resources
         set +u
         ${run_command} "${dc_run_file}" "${docker_compose_command}" "${EXTRA_DC_OPTIONS[@]}" "$@"
         set -u
         ;;
     perform_static_checks)
+        docker_engine::check_all_resources
         breeze::make_sure_precommit_is_installed
         breeze::run_static_checks "${@}"
         ;;
@@ -3470,15 +3476,19 @@ function breeze::run_breeze_command() {
         breeze::remove_images
         ;;
     perform_generate_constraints)
+        docker_engine::check_all_resources
         runs::run_generate_constraints
         ;;
     perform_prepare_airflow_packages)
+        docker_engine::check_all_resources
         build_airflow_packages::build_airflow_packages
         ;;
     perform_prepare_provider_packages)
+        docker_engine::check_all_resources
         runs::run_prepare_provider_packages "${@}"
         ;;
     perform_prepare_provider_documentation)
+        docker_engine::check_all_resources
         runs::run_prepare_provider_documentation "${@}"
         ;;
     perform_push_image)
@@ -3495,11 +3505,13 @@ function breeze::run_breeze_command() {
         breeze::setup_autocomplete
         ;;
     manage_kind_cluster)
+        docker_engine::check_all_resources
         kind::make_sure_kubernetes_tools_are_installed
         kind::get_kind_cluster_name
         kind::perform_kind_cluster_operation "${KIND_CLUSTER_OPERATION}"
         ;;
     build_docs)
+        docker_engine::check_all_resources
         runs::run_docs "${@}"
         ;;
     toggle_suppress_cheatsheet)

scripts/ci/libraries/_all_libs.sh

@@ -28,6 +28,8 @@ readonly SCRIPTS_CI_DIR
 . "${LIBRARIES_DIR}"/_traps.sh
 # shellcheck source=scripts/ci/libraries/_initialization.sh
 . "${LIBRARIES_DIR}"/_initialization.sh
+# shellcheck source=scripts/ci/libraries/_docker_engine_resources.sh
+. "${LIBRARIES_DIR}"/_docker_engine_resources.sh
 # shellcheck source=scripts/ci/libraries/_repeats.sh
 . "${LIBRARIES_DIR}"/_repeats.sh
 # shellcheck source=scripts/ci/libraries/_sanity_checks.sh

scripts/ci/libraries/_docker_engine_resources.sh

@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+function docker_engine::get_available_memory_in_docker() {
+    MEMORY_AVAILABLE_FOR_DOCKER=$(docker run --rm --entrypoint /bin/bash \
+        "${AIRFLOW_CI_IMAGE}" -c \
+        'echo $(($(getconf _PHYS_PAGES) * $(getconf PAGE_SIZE) / (1024 * 1024)))')
+    echo "${COLOR_BLUE}Memory available for Docker${COLOR_RESET}: $(numfmt --to iec $((MEMORY_AVAILABLE_FOR_DOCKER * 1024 * 1024)))"
+    export MEMORY_AVAILABLE_FOR_DOCKER
+}
+
+function docker_engine::get_available_cpus_in_docker() {
+    CPUS_AVAILABLE_FOR_DOCKER=$(docker run --rm --entrypoint /bin/bash \
+        "${AIRFLOW_CI_IMAGE}" -c \
+        'grep -cE "cpu[0-9]+" </proc/stat')
+    echo "${COLOR_BLUE}CPUS available for Docker${COLOR_RESET}: ${CPUS_AVAILABLE_FOR_DOCKER}"
+    export CPUS_AVAILABLE_FOR_DOCKER
+}
+
+function docker_engine::get_available_disk_space_in_docker() {
+    DISK_SPACE_AVAILABLE_FOR_DOCKER=$(docker run --rm --entrypoint /bin/bash \
+        "${AIRFLOW_CI_IMAGE}" -c \
+        'df  / | tail -1 | awk '\''{print $4}'\')
+    echo "${COLOR_BLUE}Disk space available for Docker${COLOR_RESET}: $(numfmt --to iec $((DISK_SPACE_AVAILABLE_FOR_DOCKER * 1024)))"
+    export DISK_SPACE_AVAILABLE_FOR_DOCKER
+}
+
+function docker_engine::check_enough_resources() {
+    local successful_resource_check="true"
+    if (( MEMORY_AVAILABLE_FOR_DOCKER < 4000 )) ; then
+        successful_resource_check="false"
+        echo
+        echo "${COLOR_RED}WARNING! Not enough memory to use breeze. At least 4GB memory is required for Docker engine to run Breeze${COLOR_RESET}"
+    fi
+
+    if (( CPUS_AVAILABLE_FOR_DOCKER < 2 )) ; then
+        successful_resource_check="false"
+        echo
+        echo "${COLOR_RED}WARNING! Not enough CPUs to use breeze. At least 2 CPUS are required for Docker engine to run Breeze.${COLOR_RESET}"
+    fi
+
+    if (( DISK_SPACE_AVAILABLE_FOR_DOCKER < 40000000 )) ; then
+        successful_resource_check="false"
+        echo
+        echo "${COLOR_RED}WARNING! Not enough disk space to use breeze. At least 40GB are required for Docker engine to run Breeze.${COLOR_RESET}"
+    fi
+
+    if [[ ${successful_resource_check} != "true" ]];then
+        echo
+        echo "${COLOR_RED}Please check https://github.com/apache/airflow/blob/master/BREEZE.rst#resources-required for details${COLOR_RESET}"
+        echo
+    fi
+}
+
+function docker_engine::check_all_resources() {
+    docker_engine::get_available_memory_in_docker
+    docker_engine::get_available_cpus_in_docker
+    docker_engine::get_available_disk_space_in_docker
+    docker_engine::check_enough_resources
+}

scripts/ci/testing/ci_run_airflow_testing.sh

@@ -140,6 +140,11 @@ build_images::rebuild_ci_image_if_needed_with_group
 prepare_tests_to_run
 
 
+docker_engine::get_available_memory_in_docker
+docker_engine::get_available_cpus_in_docker
+docker_engine::get_available_disk_space_in_docker
+
+
 for TEST_TYPE in ${TEST_TYPES}
 do
     start_end::group_start "Running tests ${TEST_TYPE}"
@@ -170,5 +175,10 @@ do
     echo "**********************************************************************************************"
 
     run_airflow_testing_in_docker "${@}"
+
+    docker_engine::get_available_memory_in_docker
+    docker_engine::get_available_cpus_in_docker
+    docker_engine::get_available_disk_space_in_docker
+
     start_end::group_end
 done