From 3ffd21745d25e6239254fe3f5688b34f5f6f77e8 Mon Sep 17 00:00:00 2001 From: Kaxil Naik Date: Mon, 8 Feb 2021 21:56:46 +0000 Subject: [PATCH] Remove reinstalling azure-storage steps from CI / Breeze (#14102) Since https://github.com/apache/airflow/pull/12188 was merged I don't think we need this steps. This step also caused the docker build step for 2.0.1rc2 to fail Co-authored-by: Jarek Potiuk --- scripts/docker/install_airflow.sh | 7 +----- scripts/in_container/_in_container_utils.sh | 22 ++++--------------- scripts/in_container/entrypoint_ci.sh | 4 ++-- scripts/in_container/run_ci_tests.sh | 2 -- .../run_install_and_test_provider_packages.sh | 5 ++--- .../run_prepare_provider_documentation.sh | 1 - setup.py | 18 +++++---------- 7 files changed, 14 insertions(+), 45 deletions(-) diff --git a/scripts/docker/install_airflow.sh b/scripts/docker/install_airflow.sh index 15187829dc..7fd6e80a79 100755 --- a/scripts/docker/install_airflow.sh +++ b/scripts/docker/install_airflow.sh @@ -64,9 +64,7 @@ function install_airflow() { pip install ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" fi - # Work around to install azure-storage-blob - pip uninstall azure-storage azure-storage-blob azure-storage-file --yes - pip install azure-storage-blob azure-storage-file + # make sure correct PIP version is used pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" pip check || ${CONTINUE_ON_PIP_CHECK_FAILURE} @@ -83,9 +81,6 @@ function install_airflow() { pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade --upgrade-strategy only-if-needed \ ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \ - # Work around to install azure-storage-blob - pip uninstall azure-storage azure-storage-blob azure-storage-file --yes - pip install azure-storage-blob azure-storage-file # make sure correct PIP version is used pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" pip check || ${CONTINUE_ON_PIP_CHECK_FAILURE} diff --git a/scripts/in_container/_in_container_utils.sh b/scripts/in_container/_in_container_utils.sh index d685d52ac9..58f4b1d166 100644 --- a/scripts/in_container/_in_container_utils.sh +++ b/scripts/in_container/_in_container_utils.sh @@ -275,7 +275,7 @@ function install_airflow_from_wheel() { >&2 echo exit 4 fi - pip install "${airflow_package}${1}" + pip install "${airflow_package}${extras}" } function install_airflow_from_sdist() { @@ -292,20 +292,7 @@ function install_airflow_from_sdist() { >&2 echo exit 4 fi - pip install "${airflow_package}${1}" -} - -function reinstall_azure_storage_blob() { - group_start "Reinstalls azure-storage-blob (temporary workaround)" - # Reinstall azure-storage-blob here until https://github.com/apache/airflow/pull/12188 is solved - # Azure-storage-blob need to be reinstalled to overwrite azure-storage-blob installed by old version - # of the `azure-storage` library - echo - echo "Reinstalling azure-storage-blob" - echo - pip uninstall azure-storage azure-storage-blob azure-storage-file --yes - pip install azure-storage-blob azure-storage-file --no-deps --force-reinstall - group_end + pip install "${airflow_package}${extras}" } function install_remaining_dependencies() { @@ -334,13 +321,12 @@ function uninstall_airflow_and_providers() { function install_released_airflow_version() { local version="${1}" - local extras="${2}" echo - echo "Installing released ${version} version of airflow with extras ${extras}" + echo "Installing released ${version} version of airflow without extras" echo rm -rf "${AIRFLOW_SOURCES}"/*.egg-info - pip install --upgrade "apache-airflow${extras}==${version}" + pip install --upgrade "apache-airflow==${version}" } function install_all_provider_packages_from_wheels() { diff --git a/scripts/in_container/entrypoint_ci.sh b/scripts/in_container/entrypoint_ci.sh index 1c740dfc9d..ed26bdfa96 100755 --- a/scripts/in_container/entrypoint_ci.sh +++ b/scripts/in_container/entrypoint_ci.sh @@ -96,9 +96,9 @@ elif [[ ${INSTALL_AIRFLOW_VERSION} == "sdist" ]]; then uninstall_providers else echo - echo "Install airflow from PyPI including [${AIRFLOW_EXTRAS}] extras" + echo "Install airflow from PyPI without extras" echo - install_released_airflow_version "${INSTALL_AIRFLOW_VERSION}" "[${AIRFLOW_EXTRAS}]" + install_released_airflow_version "${INSTALL_AIRFLOW_VERSION}" fi if [[ ${INSTALL_PACKAGES_FROM_DIST=} == "true" ]]; then echo diff --git a/scripts/in_container/run_ci_tests.sh b/scripts/in_container/run_ci_tests.sh index 43be453338..ca3c41d540 100755 --- a/scripts/in_container/run_ci_tests.sh +++ b/scripts/in_container/run_ci_tests.sh @@ -18,8 +18,6 @@ # shellcheck source=scripts/in_container/_in_container_script_init.sh . "$( dirname "${BASH_SOURCE[0]}" )/_in_container_script_init.sh" -reinstall_azure_storage_blob - echo echo "Starting the tests with those pytest arguments:" "${@}" echo diff --git a/scripts/in_container/run_install_and_test_provider_packages.sh b/scripts/in_container/run_install_and_test_provider_packages.sh index 9b951c7d50..76d41e40d3 100755 --- a/scripts/in_container/run_install_and_test_provider_packages.sh +++ b/scripts/in_container/run_install_and_test_provider_packages.sh @@ -67,9 +67,9 @@ function install_airflow_as_specified() { uninstall_providers else echo - echo "Install airflow from PyPI including [${AIRFLOW_EXTRAS}] extras" + echo "Install airflow from PyPI without extras" echo - install_released_airflow_version "${INSTALL_AIRFLOW_VERSION}" "[${AIRFLOW_EXTRAS}]" + install_released_airflow_version "${INSTALL_AIRFLOW_VERSION}" uninstall_providers fi group_end @@ -197,7 +197,6 @@ setup_provider_packages verify_parameters install_airflow_as_specified install_remaining_dependencies -reinstall_azure_storage_blob install_provider_packages import_all_provider_classes diff --git a/scripts/in_container/run_prepare_provider_documentation.sh b/scripts/in_container/run_prepare_provider_documentation.sh index e593997537..c33abb87d7 100755 --- a/scripts/in_container/run_prepare_provider_documentation.sh +++ b/scripts/in_container/run_prepare_provider_documentation.sh @@ -104,7 +104,6 @@ install_supported_pip_version # install extra packages missing in devel_ci # TODO: remove it when devel_all == devel_ci install_remaining_dependencies -reinstall_azure_storage_blob if [[ ${BACKPORT_PACKAGES} != "true" ]]; then import_all_provider_classes diff --git a/setup.py b/setup.py index 44be3e5579..76b5bded87 100644 --- a/setup.py +++ b/setup.py @@ -218,6 +218,8 @@ azure = [ 'azure-mgmt-containerinstance>=1.5.0,<2.0', 'azure-mgmt-datalake-store>=0.5.0', 'azure-mgmt-resource>=2.2.0', + 'azure-storage-blob>=12.7.0', + 'azure-storage-common>=2.1.0', 'azure-storage-file>=2.1.0', ] cassandra = [ @@ -422,19 +424,9 @@ slack = [ 'slack_sdk>=3.0.0,<4.0.0', ] snowflake = [ - # The `azure` provider uses legacy `azure-storage` library, where `snowflake` uses the - # newer and more stable versions of those libraries. Most of `azure` operators and hooks work - # fine together with `snowflake` because the deprecated library does not overlap with the - # new libraries except the `blob` classes. So while `azure` works fine for most cases - # blob is the only exception - # Solution to that is being worked on in https://github.com/apache/airflow/pull/12188 - # once it is merged, we can move those two back to `azure` extra. - 'azure-core>=1.10.0', - 'azure-storage-blob', - 'azure-storage-common', - # Snowflake conector > 2.3.8 is needed because it has vendored urrllib3 and requests libraries which - # are monkey-patched. In earlier versions of the library, monkeypatching the libraries by snowflake - # caused other providers to fail (Google, Amazon etc.) + # Snowflake connector > 2.3.8 is needed because it has vendored-in, patched urllib and requests libraries + # In earlier versions of the snowflake library, monkey-patching the libraries caused other + # providers to fail (Google, Amazon etc.) 'snowflake-connector-python>=2.3.8', 'snowflake-sqlalchemy>=1.1.0', ]