Provider packages are installed by default in production image (#12154)

This is a fix to a problem introduced in #10806. The change
turned provider packages into namespace packages - which made
them ignored by find_packages function from setup tools - thus
prodiuction image build automatically and used by Kubernetes
tests did not have the provider packages installed.

This PR fixes it and adds future protection during CI tests of
production image to make sure that provider packages are
actually installed.

Fixes #12150

scripts/ci/images/ci_prepare_prod_image_on_ci.sh

@@ -48,4 +48,61 @@ function build_prod_images_on_ci() {
 }
 
 
+
+function verify_prod_image_has_airflow_and_providers {
+    echo
+    echo "Airflow folders installed in the image:"
+    echo
+
+    AIRFLOW_INSTALLATION_LOCATION="/home/airflow/.local"
+
+    docker run --rm --entrypoint /bin/bash "${AIRFLOW_PROD_IMAGE}" -c \
+        'find '"${AIRFLOW_INSTALLATION_LOCATION}"'/lib/python*/site-packages/airflow/ -type d'
+
+    EXPECTED_MIN_AIRFLOW_DIRS_COUNT="60"
+    readonly EXPECTED_MIN_AIRFLOW_DIRS_COUNT
+
+    COUNT_AIRFLOW_DIRS=$(docker run --rm --entrypoint /bin/bash "${AIRFLOW_PROD_IMAGE}" -c \
+         'find '"${AIRFLOW_INSTALLATION_LOCATION}"'/lib/python*/site-packages/airflow/ -type d | grep -c -v "/airflow/providers"')
+
+    echo
+    echo "Number of airflow dirs: ${COUNT_AIRFLOW_DIRS}"
+    echo
+
+    if [[ "${COUNT_AIRFLOW_DIRS}" -lt "${EXPECTED_MIN_AIRFLOW_DIRS_COUNT}" ]]; then
+        >&2 echo
+        >&2 echo Number of airflow folders installed is less than ${EXPECTED_MIN_AIRFLOW_DIRS_COUNT}
+        >&2 echo This is unexpected. Please investigate, looking at the output above!
+        >&2 echo
+        exit 1
+    else
+        echo
+        echo "OK. Airflow seems to be installed!"
+        echo
+    fi
+
+    EXPECTED_MIN_PROVIDERS_DIRS_COUNT="240"
+    readonly EXPECTED_MIN_PROVIDERS_DIRS_COUNT
+
+    COUNT_AIRFLOW_PROVIDERS_DIRS=$(docker run --rm --entrypoint /bin/bash "${AIRFLOW_PROD_IMAGE}" -c \
+         'find '"${AIRFLOW_INSTALLATION_LOCATION}"'/lib/python*/site-packages/airflow/providers -type d | grep -c "" | xargs')
+
+    echo
+    echo "Number of providers dirs: ${COUNT_AIRFLOW_PROVIDERS_DIRS}"
+    echo
+
+    if [ "${COUNT_AIRFLOW_PROVIDERS_DIRS}" -lt "${EXPECTED_MIN_PROVIDERS_DIRS_COUNT}" ]; then
+        >&2 echo
+        >&2 echo Number of providers folders installed is less than ${EXPECTED_MIN_PROVIDERS_DIRS_COUNT}
+        >&2 echo This is unexpected. Please investigate, looking at the output above!
+        >&2 echo
+        exit 1
+    else
+        echo
+        echo "OK. Airflow Providers seems to be installed!"
+        echo
+    fi
+}
+
 build_prod_images_on_ci
+verify_prod_image_has_airflow_and_providers

setup.py

@@ -27,7 +27,7 @@ from os.path import dirname
 from textwrap import wrap
 from typing import Dict, Iterable, List
 
-from setuptools import Command, find_packages, setup
+from setuptools import Command, find_namespace_packages, find_packages, setup
 
 logger = logging.getLogger(__name__)
 
@@ -771,6 +771,11 @@ def do_setup():
         else ['airflow.providers', 'airflow.providers.*']
     )
     write_version()
+    packages_to_install = (
+        find_namespace_packages(include=['airflow*'], exclude=exclude_patterns)
+        if install_providers_from_sources
+        else find_packages(include=['airflow*'], exclude=exclude_patterns)
+    )
     setup(
         name='apache-airflow',
         description='Programmatically author, schedule and monitor data pipelines',
@@ -778,7 +783,7 @@ def do_setup():
         long_description_content_type='text/markdown',
         license='Apache License 2.0',
         version=version,
-        packages=find_packages(include=['airflow*'], exclude=exclude_patterns),
+        packages=packages_to_install,
         package_data={
             'airflow': ['py.typed'],
             '': [