Production image can also be upgraded to newer dependencies (#13345)

Previously UPGRADE_TO_LATEST_CONSTRAINTS variable controlled whether the CI image uses latest dependencies rather than fixed constraints. This PR brings it also to PROD image. The name of the ARG is changed to UPGRADE_TO_NEWER_DEPENDENCIES as this corresponds better with the intention. (cherry picked from commit 82fa048c12)
2020-12-28 20:08:18 +01:00 · 2020-12-28 20:08:18 +01:00 · 643d878bc8
--- a/.github/workflows/build-images-workflow-run.yml
+++ b/.github/workflows/build-images-workflow-run.yml
@ -187,7 +187,7 @@ jobs:
      GITHUB_CONTEXT: ${{ toJson(github) }}
    outputs:
      pythonVersions: ${{ steps.selective-checks.python-versions }}
-      upgradeToLatestConstraints: ${{ steps.selective-checks.outputs.upgrade-to-latest-constraints }}
+      upgradeToNewerDependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }}
      allPythonVersions: ${{ steps.selective-checks.outputs.all-python-versions }}
      defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }}
      run-tests: ${{ steps.selective-checks.outputs.run-tests }}
@ -253,7 +253,7 @@ jobs:
      BACKEND: postgres
      PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
      GITHUB_REGISTRY_PUSH_IMAGE_TAG: ${{ github.event.workflow_run.id }}
-      UPGRADE_TO_LATEST_CONSTRAINTS: ${{ needs.build-info.outputs.upgradeToLatestConstraints }}
+      UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
      DOCKER_CACHE: ${{ needs.cancel-workflow-runs.outputs.cacheDirective }}
    steps:
      - name: >
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@ -68,7 +68,7 @@ jobs:
      GITHUB_CONTEXT: ${{ toJson(github) }}
    outputs:
      waitForImage: ${{ steps.wait-for-image.outputs.wait-for-image }}
-      upgradeToLatestConstraints: ${{ steps.selective-checks.outputs.upgrade-to-latest-constraints }}
+      upgradeToNewerDependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }}
      pythonVersions: ${{ steps.selective-checks.outputs.python-versions }}
      pythonVersionsListAsString: ${{ steps.selective-checks.outputs.python-versions-list-as-string }}
      defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }}
@ -165,7 +165,7 @@ jobs:
    if: needs.build-info.outputs.image-build == 'true'
    env:
      BACKEND: sqlite
-      UPGRADE_TO_LATEST_CONSTRAINTS: ${{ needs.build-info.outputs.upgradeToLatestConstraints }}
+      UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
    steps:
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v2
@ -772,7 +772,7 @@ jobs:
    env:
      BACKEND: sqlite
      PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }}
-      UPGRADE_TO_LATEST_CONSTRAINTS: ${{ needs.build-info.outputs.upgradeToLatestConstraints }}
+      UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
    if: needs.build-info.outputs.image-build == 'true'
    steps:
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
--- a/.github/workflows/scheduled_quarantined.yml
+++ b/.github/workflows/scheduled_quarantined.yml
@ -30,7 +30,7 @@ env:
  SKIP_CHECK_REMOTE_IMAGE: "true"
  DB_RESET: "true"
  VERBOSE: "true"
-  UPGRADE_TO_LATEST_CONSTRAINTS: false
+  UPGRADE_TO_NEWER_DEPENDENCIES: false
  PYTHON_MAJOR_MINOR_VERSION: 3.6
  USE_GITHUB_REGISTRY: "true"
  # Since we run this build on schedule, it might be that the image has never been pushed
--- a/BREEZE.rst
+++ b/BREEZE.rst
@ -1303,6 +1303,9 @@ This is the current syntax for  `./breeze <./breeze>`_:
          packages after installing Airflow. This is useful for testing provider
          packages.

+  --upgrade-to-newer-dependencies
+          Upgrades PIP packages to latest versions available without looking at the constraints.
+
  -I, --production-image
          Use production image for entering the environment and builds (not for tests).

@ -2406,6 +2409,9 @@ This is the current syntax for  `./breeze <./breeze>`_:
          packages after installing Airflow. This is useful for testing provider
          packages.

+  --upgrade-to-newer-dependencies
+          Upgrades PIP packages to latest versions available without looking at the constraints.
+
  ****************************************************************************************************
   Credentials

--- a/CI.rst
+++ b/CI.rst
@ -247,7 +247,7 @@ You can use those variables when you try to reproduce the build locally.
 +-----------------------------------------+-------------+-------------+------------+-------------------------------------------------+
 |                                                        Image build variables                                                       |
 +-----------------------------------------+-------------+-------------+------------+-------------------------------------------------+
-| ``UPGRADE_TO_LATEST_CONSTRAINTS``       |    false    |    false    |    false   | Determines whether the build should             |
+| ``UPGRADE_TO_NEWER_DEPENDENCIES``       |    false    |    false    |    false   | Determines whether the build should             |
 |                                         |             |             |     (x)    | attempt to upgrade all                          |
 |                                         |             |             |            | PIP dependencies to latest ones matching        |
 |                                         |             |             |            | ``setup.py`` limits. This tries to replicate    |
--- a/26
+++ b/26
@ -124,14 +124,14 @@ ENV DEV_APT_COMMAND=${DEV_APT_COMMAND}
 ARG ADDITIONAL_DEV_APT_COMMAND="echo"
 ENV ADDITIONAL_DEV_APT_COMMAND=${ADDITIONAL_DEV_APT_COMMAND}

-ARG ADDITIONAL_DEV_ENV_VARS=""
+ARG ADDITIONAL_DEV_APT_ENV=""

 # Note missing man directories on debian-buster
 # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199
 # Install basic and additional apt dependencies
 RUN mkdir -pv /usr/share/man/man1 \
    && mkdir -pv /usr/share/man/man7 \
-    && export ${ADDITIONAL_DEV_ENV_VARS?} \
+    && export ${ADDITIONAL_DEV_APT_ENV?} \
    && bash -o pipefail -e -u -x -c "${DEV_APT_COMMAND}" \
    && bash -o pipefail -e -u -x -c "${ADDITIONAL_DEV_APT_COMMAND}" \
    && apt-get update \
@ -242,6 +242,9 @@ ENV INSTALL_FROM_PYPI=${INSTALL_FROM_PYPI}
 ARG INSTALL_PROVIDERS_FROM_SOURCES="false"
 ENV INSTALL_PROVIDERS_FROM_SOURCES=${INSTALL_PROVIDERS_FROM_SOURCES}

+ARG UPGRADE_TO_NEWER_DEPENDENCIES="false"
+ENV UPGRADE_TO_NEWER_DEPENDENCIES=${UPGRADE_TO_NEWER_DEPENDENCIES}
+
 WORKDIR /opt/airflow

 # remove mysql from extras if client is not installed
@ -249,11 +252,20 @@ RUN if [[ ${INSTALL_MYSQL_CLIENT} != "true" ]]; then \
        AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/mysql,}; \
    fi; \
    if [[ ${INSTALL_FROM_PYPI} == "true" ]]; then \
-        pip install --user "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_INSTALL_VERSION}" \
-            --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"; \
+        if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then \
+            pip install --user "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_INSTALL_VERSION}" \
+                --upgrade --upgrade-strategy eager; \
+        else \
+            pip install --user "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_INSTALL_VERSION}" \
+                --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"; \
+        fi; \
    fi; \
    if [[ -n "${ADDITIONAL_PYTHON_DEPS}" ]]; then \
-        pip install --user ${ADDITIONAL_PYTHON_DEPS} --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"; \
+        if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then \
+            pip install --user ${ADDITIONAL_PYTHON_DEPS} --upgrade --upgrade-strategy eager; \
+        else \
+            pip install --user ${ADDITIONAL_PYTHON_DEPS} --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"; \
+        fi; \
    fi; \
    if [[ ${INSTALL_FROM_DOCKER_CONTEXT_FILES} == "true" ]]; then \
        if ls /docker-context-files/*.{whl,tar.gz} 1> /dev/null 2>&1; then \
@ -373,14 +385,14 @@ ENV RUNTIME_APT_COMMAND=${RUNTIME_APT_COMMAND}
 ARG ADDITIONAL_RUNTIME_APT_COMMAND=""
 ENV ADDITIONAL_RUNTIME_APT_COMMAND=${ADDITIONAL_RUNTIME_APT_COMMAND}

-ARG ADDITIONAL_RUNTIME_ENV_VARS=""
+ARG ADDITIONAL_RUNTIME_APT_ENV=""

 # Note missing man directories on debian-buster
 # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199
 # Install basic and additional apt dependencies
 RUN mkdir -pv /usr/share/man/man1 \
    && mkdir -pv /usr/share/man/man7 \
-    && export ${ADDITIONAL_RUNTIME_ENV_VARS?} \
+    && export ${ADDITIONAL_RUNTIME_APT_ENV?} \
    && bash -o pipefail -e -u -x -c "${RUNTIME_APT_COMMAND}" \
    && bash -o pipefail -e -u -x -c "${ADDITIONAL_RUNTIME_APT_COMMAND}" \
    && apt-get update \
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@ -311,8 +311,8 @@ COPY setup.cfg ${AIRFLOW_SOURCES}/setup.cfg

 COPY airflow/__init__.py ${AIRFLOW_SOURCES}/airflow/__init__.py

-ARG UPGRADE_TO_LATEST_CONSTRAINTS="false"
-ENV UPGRADE_TO_LATEST_CONSTRAINTS=${UPGRADE_TO_LATEST_CONSTRAINTS}
+ARG UPGRADE_TO_NEWER_DEPENDENCIES="false"
+ENV UPGRADE_TO_NEWER_DEPENDENCIES=${UPGRADE_TO_NEWER_DEPENDENCIES}

 # The goal of this line is to install the dependencies from the most current setup.py from sources
 # This will be usually incremental small set of packages in CI optimized build, so it will be very fast
@ -321,7 +321,7 @@ ENV UPGRADE_TO_LATEST_CONSTRAINTS=${UPGRADE_TO_LATEST_CONSTRAINTS}
 # But in cron job we will install latest versions matching setup.py to see if there is no breaking change
 # and push the constraints if everything is successful
 RUN if [[ ${INSTALL_FROM_PYPI} == "true" ]]; then \
-        if [[ "${UPGRADE_TO_LATEST_CONSTRAINTS}" != "false" ]]; then \
+        if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then \
            pip install -e ".[${AIRFLOW_EXTRAS}]" --upgrade --upgrade-strategy eager; \
            pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}"; \
        else \
--- a/12
+++ b/12
@ -1146,6 +1146,12 @@ function breeze::parse_arguments() {
            echo
            shift
            ;;
+        --upgrade-to-newer-dependencies)
+            export UPGRADE_TO_NEWER_DEPENDENCIES="true"
+            echo "Upgrade packages to latest versions."
+            echo
+            shift
+            ;;
        --package-format)
            export PACKAGE_FORMAT="${2}"
            echo "Selected package type: ${PACKAGE_FORMAT}"
@ -1224,7 +1230,7 @@ function breeze::parse_arguments() {
            command_to_run="perform_generate_constraints"
            export FORCE_ANSWER_TO_QUESTIONS="yes"
            export FORCE_BUILD_IMAGES="true"
-            export UPGRADE_TO_LATEST_CONSTRAINTS="true"
+            export UPGRADE_TO_NEWER_DEPENDENCIES="true"
            shift
            ;;
        prepare-airflow-packages)
@ -2284,6 +2290,10 @@ ${FORMATTED_INSTALL_AIRFLOW_VERSIONS}
        If specified it will look for packages placed in dist folder and it will install the
        packages after installing Airflow. This is useful for testing provider
        packages.
+
+--upgrade-to-newer-dependencies
+        Upgrades PIP packages to latest versions available without looking at the constraints.
+
 "
 }

--- a/2
+++ b/2
@ -167,7 +167,7 @@ additional-extras: additional-python-deps: disable-pypi-when-building skip-insta
 dev-apt-deps: additional-dev-apt-deps: dev-apt-command: additional-dev-apt-command: additional-dev-apt-env:
 runtime-apt-deps: additional-runtime-apt-deps: runtime-apt-command: additional-runtime-apt-command: additional-runtime-apt-env:
 load-default-connections load-example-dags
-install-packages-from-dist no-rbac-ui package-format:
+install-packages-from-dist no-rbac-ui package-format: upgrade-to-newer-dependencies
 test-type:
 preserve-volumes
 "
--- a/docs/apache-airflow/production-deployment.rst
+++ b/docs/apache-airflow/production-deployment.rst
@ -539,7 +539,10 @@ The following build arguments (``--build-arg`` in docker build command) can be u
 |                                          |                                          | to install airflow from such pre-vetted  |
 |                                          |                                          | packages rather than from PyPI. For this |
 |                                          |                                          | to work, also set ``INSTALL_FROM_PYPI``  |
-|                                          |                                          | to false.                                |
+------------------------------------------+------------------------------------------+------------------------------------------+
+| ``UPGRADE_TO_NEWER_DEPENDENCIES``        | ``false``                                | If set to true, the dependencies are     |
+|                                          |                                          | upgraded to newer versions matching      |
+|                                          |                                          | setup.py before installation.            |
 +------------------------------------------+------------------------------------------+------------------------------------------+
 | ``ADDITIONAL_AIRFLOW_EXTRAS``            |                                          | Optional additional extras with which    |
 |                                          |                                          | airflow is installed.                    |
--- a/scripts/ci/libraries/_build_images.sh
+++ b/scripts/ci/libraries/_build_images.sh
@ -659,7 +659,7 @@ Docker building ${AIRFLOW_CI_IMAGE}.
        --build-arg ADDITIONAL_RUNTIME_APT_ENV="${ADDITIONAL_RUNTIME_APT_ENV}" \
        --build-arg INSTALL_FROM_PYPI="${INSTALL_FROM_PYPI}" \
        --build-arg INSTALL_FROM_DOCKER_CONTEXT_FILES="${INSTALL_FROM_DOCKER_CONTEXT_FILES}" \
-        --build-arg UPGRADE_TO_LATEST_CONSTRAINTS="${UPGRADE_TO_LATEST_CONSTRAINTS}" \
+        --build-arg UPGRADE_TO_NEWER_DEPENDENCIES="${UPGRADE_TO_NEWER_DEPENDENCIES}" \
        --build-arg BUILD_ID="${CI_BUILD_ID}" \
        --build-arg COMMIT_SHA="${COMMIT_SHA}" \
        "${additional_dev_args[@]}" \
@ -794,6 +794,7 @@ function build_images::build_prod_images() {
        --build-arg AIRFLOW_PRE_CACHED_PIP_PACKAGES="${AIRFLOW_PRE_CACHED_PIP_PACKAGES}" \
        --build-arg INSTALL_FROM_PYPI="${INSTALL_FROM_PYPI}" \
        --build-arg INSTALL_FROM_DOCKER_CONTEXT_FILES="${INSTALL_FROM_DOCKER_CONTEXT_FILES}" \
+        --build-arg UPGRADE_TO_NEWER_DEPENDENCIES="${UPGRADE_TO_NEWER_DEPENDENCIES}" \
        --build-arg BUILD_ID="${CI_BUILD_ID}" \
        --build-arg COMMIT_SHA="${COMMIT_SHA}" \
        "${DOCKER_CACHE_PROD_BUILD_DIRECTIVE[@]}" \
@ -824,6 +825,7 @@ function build_images::build_prod_images() {
        --build-arg AIRFLOW_PRE_CACHED_PIP_PACKAGES="${AIRFLOW_PRE_CACHED_PIP_PACKAGES}" \
        --build-arg INSTALL_FROM_PYPI="${INSTALL_FROM_PYPI}" \
        --build-arg INSTALL_FROM_DOCKER_CONTEXT_FILES="${INSTALL_FROM_DOCKER_CONTEXT_FILES}" \
+        --build-arg UPGRADE_TO_NEWER_DEPENDENCIES="${UPGRADE_TO_NEWER_DEPENDENCIES}" \
        --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \
        --build-arg AIRFLOW_BRANCH="${AIRFLOW_BRANCH_FOR_PYPI_PRELOADING}" \
        --build-arg AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS}" \
--- a/scripts/ci/libraries/_initialization.sh
+++ b/scripts/ci/libraries/_initialization.sh
@ -362,7 +362,7 @@ function initialization::initialize_image_build_variables() {

    # By default we are not upgrading to latest version of constraints when building Docker CI image
    # This will only be done in cron jobs
-    export UPGRADE_TO_LATEST_CONSTRAINTS=${UPGRADE_TO_LATEST_CONSTRAINTS:="false"}
+    export UPGRADE_TO_NEWER_DEPENDENCIES=${UPGRADE_TO_NEWER_DEPENDENCIES:="false"}

    # Checks if the image should be rebuilt
    export CHECK_IMAGE_FOR_REBUILD="${CHECK_IMAGE_FOR_REBUILD:="true"}"
@ -615,7 +615,7 @@ Verbosity variables:

 Image build variables:

-    UPGRADE_TO_LATEST_CONSTRAINTS: ${UPGRADE_TO_LATEST_CONSTRAINTS}
+    UPGRADE_TO_NEWER_DEPENDENCIES: ${UPGRADE_TO_NEWER_DEPENDENCIES}
    CHECK_IMAGE_FOR_REBUILD: ${CHECK_IMAGE_FOR_REBUILD}


--- a/scripts/ci/libraries/_runs.sh
+++ b/scripts/ci/libraries/_runs.sh
@ -30,8 +30,12 @@ function runs::run_docs() {
 # Downloads packages from PIP
 function runs::run_pip_download() {
    start_end::group_start "PIP download"
-    pip_download_command="pip download -d /dist '.[${INSTALLED_EXTRAS}]' --constraint 'https://raw.githubusercontent.com/apache/airflow/${DEFAULT_CONSTRAINTS_BRANCH}/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt'"
-
+    if [[ ${UPGRADE_TO_NEWER_DEPENDENCIES} ]]; then
+        pip_download_command="pip download -d /dist '.[${INSTALLED_EXTRAS}]'"
+    else
+        pip_download_command="pip download -d /dist '.[${INSTALLED_EXTRAS}]' --constraint
+'https://raw.githubusercontent.com/apache/airflow/${DEFAULT_CONSTRAINTS_BRANCH}/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt'"
+    fi
    # Download all dependencies needed
    docker run --rm --entrypoint /bin/bash \
        "${EXTRA_DOCKER_FLAGS[@]}" \
--- a/scripts/ci/selective_ci_checks.sh
+++ b/scripts/ci/selective_ci_checks.sh
@ -46,24 +46,24 @@ if [[ ${PR_LABELS=} == *"upgrade to newer dependencies"* ]]; then
    echo
    echo "Found the right PR labels in '${PR_LABELS=}': 'upgrade to newer dependencies''"
    echo
-    UPGRADE_TO_LATEST_CONSTRAINTS_LABEL="true"
+    UPGRADE_TO_NEWER_DEPENDENCIES_LABEL="true"
 else
    echo
    echo "Did not find the right PR labels in '${PR_LABELS=}': 'upgrade to newer dependencies'"
    echo
-    UPGRADE_TO_LATEST_CONSTRAINTS_LABEL="false"
+    UPGRADE_TO_NEWER_DEPENDENCIES_LABEL="false"
 fi

 function output_all_basic_variables() {
-    if [[ "${UPGRADE_TO_LATEST_CONSTRAINTS_LABEL}" == "true" ||
+    if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES_LABEL}" == "true" ||
            ${EVENT_NAME} == 'push' || ${EVENT_NAME} == "scheduled" ]]; then
        # Trigger upgrading to latest constraints where label is set or when
        # SHA of the merge commit triggers rebuilding layer in the docker image
        # Each build that upgrades to latest constraints will get truly latest constraints, not those
        # Cached in the image this way
-        initialization::ga_output upgrade-to-latest-constraints "${INCOMING_COMMIT_SHA}"
+        initialization::ga_output upgrade-to-newer-dependencies "${INCOMING_COMMIT_SHA}"
    else
-        initialization::ga_output upgrade-to-latest-constraints "false"
+        initialization::ga_output upgrade-to-newer-dependencies "false"
    fi

    if [[ ${FULL_TESTS_NEEDED_LABEL} == "true" ]]; then