Install airflow and providers from dist and verifies them (#13033)

* Install airflow and providers from dist and verifies them

This check is there to prevent problems similar to those reported
in #13027 and fixed in #13031.

Previously we always built airflow from wheels, only providers were
installed from sdist packages and tested. In this version both
airflow and providers are installed using the same package format
(sdist or wheel).

* Update scripts/in_container/entrypoint_ci.sh

Co-authored-by: Kaxil Naik <kaxilnaik@gmail.com>

Co-authored-by: Kaxil Naik <kaxilnaik@gmail.com>

.github/workflows/ci.yml

@@ -210,6 +210,8 @@ jobs:
       - name: "Free space"
         run: ./scripts/ci/tools/ci_free_space_on_ci.sh
         if: needs.build-info.outputs.waitForImage == 'true'
+      - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
+        run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
       - name: "Verify CI image Py${{matrix.python-version}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
         run: ./scripts/ci/images/ci_verify_ci_image.sh
         if: needs.build-info.outputs.waitForImage == 'true'
@@ -358,16 +360,20 @@ jobs:
 
   prepare-backport-provider-packages:
     timeout-minutes: 30
-    name: "Backport packages"
+    name: "Backport packages: ${{ matrix.package-format }}"
     runs-on: ubuntu-20.04
     needs: [build-info, ci-images]
+    strategy:
+      matrix:
+        package-format: ['wheel', 'sdist']
     env:
       # In this case we want to install airflow from the latest released 1.10 version
       # all provider packages are installed from wheels or .tar.gz files
       INSTALL_AIRFLOW_VERSION: "1.10.14"
       PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
       BACKPORT_PACKAGES: "true"
-      VERSION_SUFFIX_FOR_SVN: "rc1"
+      VERSION_SUFFIX_FOR_PYPI: "rc1"
+      PACKAGE_FORMAT: ${{ matrix.package-format }}
     if: needs.build-info.outputs.image-build == 'true'
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -382,18 +388,10 @@ jobs:
         run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
       - name: "Prepare provider readmes"
         run: ./scripts/ci/provider_packages/ci_prepare_provider_readmes.sh
-      - name: "Prepare provider packages"
+      - name: "Prepare provider packages: ${{ matrix.package-format }}"
         run: ./scripts/ci/provider_packages/ci_prepare_provider_packages.sh
-        env:
-          PACKAGE_FORMAT: "both"
-      - name: "Install and test provider packages via wheel files"
+      - name: "Install and test provider packages and airflow via ${{ matrix.package-format }} files"
         run: ./scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh
-        env:
-          PACKAGE_FORMAT: "wheel"
-      - name: "Install and test provider packages via sdist files"
-        run: ./scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh
-        env:
-          PACKAGE_FORMAT: "sdist"
       - name: "Upload provider package artifacts"
         uses: actions/upload-artifact@v2
         if: always()
@@ -403,7 +401,7 @@ jobs:
           retention-days: 7
       - name: "Upload readme artifacts"
         uses: actions/upload-artifact@v2
-        if: always()
+        if: always() && matrix.package-format == 'wheel'
         with:
           name: airflow-backport-readmes
           path: "./files/airflow-readme-*"
@@ -411,15 +409,17 @@ jobs:
 
   prepare-provider-packages:
     timeout-minutes: 30
-    name: "Provider packages"
+    name: "Provider packages ${{ matrix.package-format }}"
     runs-on: ubuntu-20.04
     needs: [build-info, ci-images]
     env:
-      INSTALL_AIRFLOW_VERSION: "wheel"
+      INSTALL_AIRFLOW_VERSION: "${{ matrix.package-format }}"
       PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
-      VERSION_SUFFIX_FOR_PYPI: "a2"
-      VERSION_SUFFIX_FOR_SVN: "a2"
-      PACKAGE_FORMAT: "both"
+      VERSION_SUFFIX_FOR_PYPI: "rc1"
+      PACKAGE_FORMAT: ${{ matrix.package-format }}
+    strategy:
+      matrix:
+        package-format: ['wheel', 'sdist']
     if: needs.build-info.outputs.image-build == 'true'
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -434,20 +434,12 @@ jobs:
         run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
       - name: "Prepare provider readmes"
         run: ./scripts/ci/provider_packages/ci_prepare_provider_readmes.sh
-      - name: "Prepare provider packages"
+      - name: "Prepare provider packages: ${{ matrix.package-format }}"
         run: ./scripts/ci/provider_packages/ci_prepare_provider_packages.sh
-        env:
-          PACKAGE_FORMAT: "both"
-      - name: "Prepare airflow package so that it can be installed"
+      - name: "Prepare airflow packages: ${{ matrix.package-format }}"
         run: ./scripts/ci/build_airflow/ci_build_airflow_package.sh
-      - name: "Install and test provider packages via wheel files"
+      - name: "Install and test provider packages and airflow via ${{ matrix.package-format }} files"
         run: ./scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh
-        env:
-          PACKAGE_FORMAT: "wheel"
-      - name: "Install and test provider packages via sdist files"
-        run: ./scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh
-        env:
-          PACKAGE_FORMAT: "sdist"
       - name: "Upload provider package artifacts"
         uses: actions/upload-artifact@v2
         if: always()
@@ -457,7 +449,7 @@ jobs:
           retention-days: 7
       - name: "Upload readme artifacts"
         uses: actions/upload-artifact@v2
-        if: always()
+        if: always() && matrix.package-format == 'wheel'
         with:
           name: airflow-provider-readmes
           path: "./files/airflow-readme-*"
@@ -821,6 +813,8 @@ jobs:
       - name: "Free space"
         run: ./scripts/ci/tools/ci_free_space_on_ci.sh
         if: needs.build-info.outputs.waitForImage == 'true'
+      - name: "Prepare PROD Image"
+        run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh
       - name: "Verify PROD image Py${{matrix.python-version}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
         run: ./scripts/ci/images/ci_verify_prod_image.sh
         if: needs.build-info.outputs.waitForImage == 'true'

BREEZE.rst

@@ -1275,7 +1275,7 @@ This is the current syntax for  `./breeze <./breeze>`_:
           If specified, installs Airflow directly from PIP released version. This happens at
           image building time in production image and at container entering time for CI image. One of:
 
-                 1.10.14 1.10.12 1.10.11 1.10.10 1.10.9 none wheel
+                 1.10.14 1.10.12 1.10.11 1.10.10 1.10.9 none wheel sdist
 
           When 'none' is used, you can install airflow from local packages. When building image,
           airflow package should be added to 'docker-context-files' and
@@ -2378,7 +2378,7 @@ This is the current syntax for  `./breeze <./breeze>`_:
           If specified, installs Airflow directly from PIP released version. This happens at
           image building time in production image and at container entering time for CI image. One of:
 
-                 1.10.14 1.10.12 1.10.11 1.10.10 1.10.9 none wheel
+                 1.10.14 1.10.12 1.10.11 1.10.10 1.10.9 none wheel sdist
 
           When 'none' is used, you can install airflow from local packages. When building image,
           airflow package should be added to 'docker-context-files' and

Dockerfile

@@ -255,7 +255,7 @@ RUN if [[ ${INSTALL_MYSQL_CLIENT} != "true" ]]; then \
         pip install --user ${ADDITIONAL_PYTHON_DEPS} --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"; \
     fi; \
     if [[ ${INSTALL_FROM_DOCKER_CONTEXT_FILES} == "true" ]]; then \
-        if ls /docker-context-files/*.whl 1> /dev/null 2>&1; then \
+        if ls /docker-context-files/*.{whl,tar.gz} 1> /dev/null 2>&1; then \
             pip install --user --no-deps /docker-context-files/*.{whl,tar.gz}; \
         fi ; \
     fi; \

breeze-complete

@@ -58,6 +58,7 @@ _breeze_allowed_install_airflow_versions=$(cat <<-EOF
 1.10.9
 none
 wheel
+sdist
 EOF
 )
 

docs/apache-airflow/production-deployment.rst

@@ -618,6 +618,7 @@ production image. There are three types of build:
 |                                   |                        | GitHub repository tag or branch or "." to install from sources.                   |
 |                                   |                        | Note that installing from local sources requires appropriate values of the        |
 |                                   |                        | ``AIRFLOW_SOURCES_FROM`` and ``AIRFLOW_SOURCES_TO`` variables as described below. |
+|                                   |                        | Only used when ``INSTALL_FROM_PYPI`` is set to ``true``.                          |
 +-----------------------------------+------------------------+-----------------------------------------------------------------------------------+
 | ``AIRFLOW_INSTALL_VERSION``       |                        | Optional - might be used for package installation of different Airflow version    |
 |                                   |                        | for example"==1.10.14". For consistency, you should also set``AIRFLOW_VERSION``   |

scripts/ci/images/ci_verify_ci_image.sh

@@ -52,4 +52,6 @@ function pull_ci_image() {
 
 build_images::prepare_ci_build
 
+pull_ci_image
+
 verify_ci_image_dependencies

scripts/ci/images/ci_verify_prod_image.sh

@@ -90,6 +90,8 @@ function pull_prod_image() {
 
 build_images::prepare_prod_build
 
+pull_prod_image
+
 verify_prod_image_has_airflow_and_providers
 
 verify_prod_image_dependencies

scripts/ci/libraries/_build_images.sh

@@ -921,16 +921,11 @@ function build_images::build_prod_images_from_packages() {
     # Build necessary provider packages
     runs::run_prepare_provider_packages "${INSTALLED_PROVIDERS[@]}"
 
-    mv "${AIRFLOW_SOURCES}/dist/"*.whl "${AIRFLOW_SOURCES}/docker-context-files/"
+    mv "${AIRFLOW_SOURCES}/dist/"* "${AIRFLOW_SOURCES}/docker-context-files/"
 
     # Build apache airflow packages
     build_airflow_packages::build_airflow_packages
 
-    # Remove generated tar.gz packages
-    rm -f "${AIRFLOW_SOURCES}/dist/"apache-airflow*.tar.gz
-
-    # move the packages to docker-context-files folder
-    mkdir -pv "${AIRFLOW_SOURCES}/docker-context-files"
     mv "${AIRFLOW_SOURCES}/dist/"* "${AIRFLOW_SOURCES}/docker-context-files/"
     build_images::build_prod_images
 }

scripts/in_container/_in_container_utils.sh

@@ -272,7 +272,7 @@ function install_airflow_from_wheel() {
     local extras
     extras="${1}"
     local airflow_package
-    airflow_package=$(find /dist/ -maxdepth 1 -type f -name 'apache_airflow-*.whl')
+    airflow_package=$(find /dist/ -maxdepth 1 -type f -name 'apache_airflow-[0-9]*.whl')
     echo
     echo "Found package: ${airflow_package}. Installing."
     echo
@@ -285,6 +285,23 @@ function install_airflow_from_wheel() {
     pip install "${airflow_package}${1}" >"${OUTPUT_PRINTED_ONLY_ON_ERROR}" 2>&1
 }
 
+function install_airflow_from_sdist() {
+    local extras
+    extras="${1}"
+    local airflow_package
+    airflow_package=$(find /dist/ -maxdepth 1 -type f -name 'apache-airflow-[0-9]*.tar.gz')
+    echo
+    echo "Found package: ${airflow_package}. Installing."
+    echo
+    if [[ -z "${airflow_package}" ]]; then
+        >&2 echo
+        >&2 echo "ERROR! Could not find airflow sdist package to install in dist"
+        >&2 echo
+        exit 4
+    fi
+    pip install "${airflow_package}${1}" >"${OUTPUT_PRINTED_ONLY_ON_ERROR}" 2>&1
+}
+
 function install_remaining_dependencies() {
     pip install apache-beam[gcp] >"${OUTPUT_PRINTED_ONLY_ON_ERROR}" 2>&1
 }

scripts/in_container/entrypoint_ci.sh

@@ -103,7 +103,7 @@ if [[ -z ${INSTALL_AIRFLOW_VERSION=} ]]; then
     export PYTHONPATH=${AIRFLOW_SOURCES}
 elif [[ ${INSTALL_AIRFLOW_VERSION} == "none"  ]]; then
     echo
-    echo "Skip installing airflow - only install wheel packages that are present locally"
+    echo "Skip installing airflow - only install wheel/tar.gz packages that are present locally"
     echo
     uninstall_airflow_and_providers
 elif [[ ${INSTALL_AIRFLOW_VERSION} == "wheel"  ]]; then
@@ -113,6 +113,13 @@ elif [[ ${INSTALL_AIRFLOW_VERSION} == "wheel"  ]]; then
     uninstall_airflow_and_providers
     install_airflow_from_wheel "[all]"
     uninstall_providers
+elif [[ ${INSTALL_AIRFLOW_VERSION} == "sdist"  ]]; then
+    echo
+    echo "Install airflow from sdist package with [all] extras but uninstalling providers."
+    echo
+    uninstall_airflow_and_providers
+    install_airflow_from_sdist "[all]"
+    uninstall_providers
 else
     echo
     echo "Install airflow from PyPI including [all] extras"

scripts/in_container/run_install_and_test_provider_packages.sh

@@ -53,11 +53,20 @@ elif [[ ${INSTALL_AIRFLOW_VERSION} == "wheel"  ]]; then
     echo
     uninstall_airflow_and_providers
     install_airflow_from_wheel "[all]"
+    uninstall_providers
+elif [[ ${INSTALL_AIRFLOW_VERSION} == "sdist"  ]]; then
+    echo
+    echo "Install airflow from sdist including [all] extras"
+    echo
+    uninstall_airflow_and_providers
+    install_airflow_from_sdist "[all]"
+    uninstall_providers
 else
     echo
     echo "Install airflow from PyPI including [all] extras"
     echo
     install_released_airflow_version "${INSTALL_AIRFLOW_VERSION}" "[all]"
+    uninstall_providers
 fi
 
 echo