diff --git a/.github/workflows/build-images-workflow-run.yml b/.github/workflows/build-images-workflow-run.yml index 8a921d1dc3..fc3d5a1ee9 100644 --- a/.github/workflows/build-images-workflow-run.yml +++ b/.github/workflows/build-images-workflow-run.yml @@ -258,6 +258,7 @@ jobs: PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} GITHUB_REGISTRY_PUSH_IMAGE_TAG: ${{ github.event.workflow_run.id }} UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} + CONTINUE_ON_PIP_CHECK_FAILURE: "true" DOCKER_CACHE: ${{ needs.cancel-workflow-runs.outputs.cacheDirective }} steps: - name: > diff --git a/BREEZE.rst b/BREEZE.rst index 7e3a07fa67..f78c98cdda 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -1299,6 +1299,12 @@ This is the current syntax for `./breeze <./breeze>`_: image building time in production image and at container entering time for CI image. This can be a GitHub branch like master or v1-10-test, or a tag like 2.0.0a1. + --installation-method INSTALLATION_METHOD + Method of installing airflow - either from the sources ('.') or from package + 'apache-airflow' to install from PyPI. Default in Breeze is to install from sources. One of: + + . apache-airflow + --no-rbac-ui Disables RBAC UI when Airflow 1.10.* is installed. @@ -1310,6 +1316,9 @@ This is the current syntax for `./breeze <./breeze>`_: --upgrade-to-newer-dependencies Upgrades PIP packages to latest versions available without looking at the constraints. + --continue-on-pip-check-failure + Continue even if 'pip check' fails. + -I, --production-image Use production image for entering the environment and builds (not for tests). @@ -2410,6 +2419,12 @@ This is the current syntax for `./breeze <./breeze>`_: image building time in production image and at container entering time for CI image. This can be a GitHub branch like master or v1-10-test, or a tag like 2.0.0a1. + --installation-method INSTALLATION_METHOD + Method of installing airflow - either from the sources ('.') or from package + 'apache-airflow' to install from PyPI. Default in Breeze is to install from sources. One of: + + . apache-airflow + --no-rbac-ui Disables RBAC UI when Airflow 1.10.* is installed. @@ -2421,6 +2436,9 @@ This is the current syntax for `./breeze <./breeze>`_: --upgrade-to-newer-dependencies Upgrades PIP packages to latest versions available without looking at the constraints. + --continue-on-pip-check-failure + Continue even if 'pip check' fails. + **************************************************************************************************** Credentials diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 9ef1dffb74..24ef274a24 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -391,11 +391,6 @@ Step 4: Prepare PR the "full tests needed" label is set for your PR. Additional check is set that prevents from accidental merging of the request until full matrix of tests succeeds for the PR. - * when your change has "upgrade to newer dependencies" label set, constraints will be automatically - upgraded to latest constraints matching your setup.py. This is useful in case you want to force - upgrade to a latest version of dependencies. You can ask committers to set the label for you - when you need it in your PR. - More details about the PR workflow be found in `PULL_REQUEST_WORKFLOW.rst `_. diff --git a/Dockerfile b/Dockerfile index 76508545bb..2dc297170e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -61,9 +61,6 @@ ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE} ARG PYTHON_MAJOR_MINOR_VERSION ENV PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION} -ARG AIRFLOW_PIP_VERSION -ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} - # Make sure noninteractive debian install is used and language variables set ENV DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8 LC_ALL=C.UTF-8 \ LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8 @@ -145,7 +142,8 @@ RUN mkdir -pv /usr/share/man/man1 \ ARG INSTALL_MYSQL_CLIENT="true" ENV INSTALL_MYSQL_CLIENT=${INSTALL_MYSQL_CLIENT} -COPY scripts/docker /scripts/docker +# Only copy install_mysql.sh to not invalidate cache on other script changes +COPY scripts/docker/install_mysql.sh /scripts/docker/install_mysql.sh COPY docker-context-files /docker-context-files # fix permission issue in Azure DevOps when running the script RUN chmod a+x /scripts/docker/install_mysql.sh @@ -172,6 +170,15 @@ RUN if [[ -f /docker-context-files/.pypirc ]]; then \ cp /docker-context-files/.pypirc /root/.pypirc; \ fi +ARG AIRFLOW_PIP_VERSION +ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} + +# Install Airflow with "--user" flag, so that we can copy the whole .local folder to the final image +# from the build image and always in non-editable mode +ENV AIRFLOW_INSTALL_USER_FLAG="--user" +ENV AIRFLOW_INSTALL_EDITABLE_FLAG="" + +# Upgrade to specific PIP version RUN pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}" # By default we do not use pre-cached packages, but in CI/Breeze environment we override this to speed up @@ -179,16 +186,27 @@ RUN pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}" ARG AIRFLOW_PRE_CACHED_PIP_PACKAGES="false" ENV AIRFLOW_PRE_CACHED_PIP_PACKAGES=${AIRFLOW_PRE_CACHED_PIP_PACKAGES} +# By default we install providers from PyPI but in case of Breeze build we want to install providers +# from local sources without the need of preparing provider packages upfront. This value is +# automatically overridden by Breeze scripts. +ARG INSTALL_PROVIDERS_FROM_SOURCES="false" +ENV INSTALL_PROVIDERS_FROM_SOURCES=${INSTALL_PROVIDERS_FROM_SOURCES} + +# Increase the value here to force reinstalling Apache Airflow pip dependencies +ARG PIP_DEPENDENCIES_EPOCH_NUMBER="1" +ENV PIP_DEPENDENCIES_EPOCH_NUMBER=${PIP_DEPENDENCIES_EPOCH_NUMBER} + +# Only copy install_airflow_from_latest_master.sh to not invalidate cache on other script changes +COPY scripts/docker/install_airflow_from_latest_master.sh /scripts/docker/install_airflow_from_latest_master.sh +# fix permission issue in Azure DevOps when running the script +RUN chmod a+x /scripts/docker/install_airflow_from_latest_master.sh + # In case of Production build image segment we want to pre-install master version of airflow # dependencies from GitHub so that we do not have to always reinstall it from the scratch. +# The Airflow (and providers in case INSTALL_PROVIDERS_FROM_SOURCES is "false") +# are uninstalled, only dependencies remain RUN if [[ ${AIRFLOW_PRE_CACHED_PIP_PACKAGES} == "true" ]]; then \ - if [[ ${INSTALL_MYSQL_CLIENT} != "true" ]]; then \ - AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/mysql,}; \ - fi; \ - pip install --user \ - "https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]" \ - --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" \ - && pip uninstall --yes apache-airflow; \ + /scripts/docker/install_airflow_from_latest_master.sh; \ fi # By default we install latest airflow from PyPI so we do not need to copy sources of Airflow @@ -209,6 +227,7 @@ ENV CASS_DRIVER_BUILD_CONCURRENCY=${CASS_DRIVER_BUILD_CONCURRENCY} ARG AIRFLOW_VERSION ENV AIRFLOW_VERSION=${AIRFLOW_VERSION} +# Add extra python dependencies ARG ADDITIONAL_PYTHON_DEPS="" ENV ADDITIONAL_PYTHON_DEPS=${ADDITIONAL_PYTHON_DEPS} @@ -236,91 +255,43 @@ ENV INSTALL_FROM_DOCKER_CONTEXT_FILES=${INSTALL_FROM_DOCKER_CONTEXT_FILES} ARG INSTALL_FROM_PYPI="true" ENV INSTALL_FROM_PYPI=${INSTALL_FROM_PYPI} -# By default we install providers from PyPI but in case of Breze build we want to install providers -# from local sources without the neeed of preparing provider packages upfront. This value is -# automatically overridden by Breeze scripts. -ARG INSTALL_PROVIDERS_FROM_SOURCES="false" -ENV INSTALL_PROVIDERS_FROM_SOURCES=${INSTALL_PROVIDERS_FROM_SOURCES} - +# By default we do not upgrade to latest dependencies ARG UPGRADE_TO_NEWER_DEPENDENCIES="false" ENV UPGRADE_TO_NEWER_DEPENDENCIES=${UPGRADE_TO_NEWER_DEPENDENCIES} +# Those are additional constraints that are needed for some extras but we do not want to +# Force them on the main Airflow package. +# * urllib3 - required to keep boto3 happy +ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="urllib3<1.26 chardet<4" + WORKDIR /opt/airflow +ARG CONTINUE_ON_PIP_CHECK_FAILURE="false" + +# Copy all install scripts here +COPY scripts/docker/install*.sh /scripts/docker/ +# fix permission issue in Azure DevOps when running the script +RUN chmod a+x /scripts/docker/instal*.sh + # hadolint ignore=SC2086, SC2010 -RUN if [[ ${INSTALL_MYSQL_CLIENT} != "true" ]]; then \ - # Remove mysql from extras if client is not installed \ - AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/mysql,}; \ - fi; \ - if [[ ${INSTALL_FROM_PYPI} == "true" ]]; then \ - if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then \ - pip install --user "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_INSTALL_VERSION}" \ - --upgrade --upgrade-strategy eager; \ - pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}"; \ - else \ - pip install --upgrade --upgrade-strategy only-if-needed \ - --user "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_INSTALL_VERSION}" \ - --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"; \ - pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}"; \ - fi; \ +RUN if [[ ${INSTALL_FROM_PYPI} == "true" ]]; then \ + /scripts/docker/install_airflow.sh; \ fi; \ if [[ ${INSTALL_FROM_DOCKER_CONTEXT_FILES} == "true" ]]; then \ - reinstalling_apache_airflow_package=$(ls /docker-context-files/apache?airflow?[0-9]*.{whl,tar.gz} 2>/dev/null || true); \ - if [[ "${reinstalling_apache_airflow_package}" != "" ]]; then \ - # install airflow with extras \ - reinstalling_apache_airflow_package="${reinstalling_apache_airflow_package}[${AIRFLOW_EXTRAS}]"; \ - fi; \ - reinstalling_apache_airflow_providers_packages=$(ls /docker-context-files/apache?airflow?providers*.{whl,tar.gz} 2>/dev/null || true); \ - if [[ ${reinstalling_apache_airflow_package} != "" || \ - ${reinstalling_apache_airflow_providers_packages} == "" ]]; then \ - if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then \ - pip install --force-reinstall --upgrade --upgrade-strategy eager \ - --user ${reinstalling_apache_airflow_package} \ - ${reinstalling_apache_airflow_providers_packages}; \ - pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}"; \ - else \ - pip install --force-reinstall --upgrade --upgrade-strategy only-if-needed \ - --user ${reinstalling_apache_airflow_package} \ - ${reinstalling_apache_airflow_providers_packages} \ - --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"; \ - pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}"; \ - fi; \ - fi ; \ - # All the other packages we want to reinstall as-is, without dependencies \ - reinstalling_other_packages=$(ls /docker-context-files/*.{whl,tar.gz} 2>/dev/null | \ - grep -v apache_airflow | grep -v apache-airflow || true); \ - if [[ "${reinstalling_other_packages}" != "" ]]; then \ - pip install --force-reinstall --user --no-deps ${reinstalling_other_packages}; \ - fi; \ + /scripts/docker/install_from_docker_context_files.sh; \ fi; \ if [[ -n "${ADDITIONAL_PYTHON_DEPS}" ]]; then \ - if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then \ - pip install --user ${ADDITIONAL_PYTHON_DEPS} \ - ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} \ - --upgrade --upgrade-strategy eager; \ - pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}"; \ - pip check || ${CONTINUE_ON_PIP_CHECK_FAILURE}; \ - else \ - pip install --user ${ADDITIONAL_PYTHON_DEPS} --upgrade --upgrade-strategy only-if-needed \ - --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"; \ - pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}"; \ - fi; \ + /scripts/docker/install_additional_dependencies.sh; \ fi; \ find /root/.local/ -name '*.pyc' -print0 | xargs -0 rm -r || true ; \ find /root/.local/ -type d -name '__pycache__' -print0 | xargs -0 rm -r || true -RUN AIRFLOW_SITE_PACKAGE="/root/.local/lib/python${PYTHON_MAJOR_MINOR_VERSION}/site-packages/airflow"; \ - if [[ -f "${AIRFLOW_SITE_PACKAGE}/www_rbac/package.json" ]]; then \ - WWW_DIR="${AIRFLOW_SITE_PACKAGE}/www_rbac"; \ - elif [[ -f "${AIRFLOW_SITE_PACKAGE}/www/package.json" ]]; then \ - WWW_DIR="${AIRFLOW_SITE_PACKAGE}/www"; \ - fi; \ - if [[ ${WWW_DIR:=} != "" ]]; then \ - yarn --cwd "${WWW_DIR}" install --frozen-lockfile --no-cache; \ - yarn --cwd "${WWW_DIR}" run prod; \ - rm -rf "${WWW_DIR}/node_modules"; \ - rm -vf "${WWW_DIR}"/{package.json,yarn.lock,.eslintignore,.eslintrc,.stylelintignore,.stylelintrc,compile_assets.sh,webpack.config.js} ;\ - fi +# Copy compile_www_assets.sh install scripts here +COPY scripts/docker/compile_www_assets.sh /scripts/docker/compile_www_assets.sh +# fix permission issue in Azure DevOps when running the script +RUN chmod a+x /scripts/docker/compile_www_assets.sh + +RUN /scripts/docker/compile_www_assets.sh # make sure that all directories and files in .local are also group accessible RUN find /root/.local -executable -print0 | xargs --null chmod g+x && \ @@ -440,10 +411,12 @@ RUN mkdir -pv /usr/share/man/man1 \ ARG INSTALL_MYSQL_CLIENT="true" ENV INSTALL_MYSQL_CLIENT=${INSTALL_MYSQL_CLIENT} -COPY scripts/docker /scripts/docker -# fix permission issue in Azure DevOps when running the script +# Only copy install_mysql script. We do not need any other +COPY scripts/docker/install_mysql.sh /scripts/docker/install_mysql.sh +# fix permission issue in Azure DevOps when running the scripts RUN chmod a+x /scripts/docker/install_mysql.sh -RUN ./scripts/docker/install_mysql.sh prod + +RUN /scripts/docker/install_mysql.sh prod ENV AIRFLOW_UID=${AIRFLOW_UID} ENV AIRFLOW_GID=${AIRFLOW_GID} diff --git a/Dockerfile.ci b/Dockerfile.ci index 125843756a..b18656bdc3 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -29,9 +29,6 @@ ENV AIRFLOW_VERSION=$AIRFLOW_VERSION ARG PYTHON_MAJOR_MINOR_VERSION="3.6" ENV PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION} -ARG AIRFLOW_PIP_VERSION=20.2.4 -ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} - # Print versions RUN echo "Base image: ${PYTHON_BASE_IMAGE}" RUN echo "Airflow version: ${AIRFLOW_VERSION}" @@ -108,10 +105,11 @@ RUN mkdir -pv /usr/share/man/man1 \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -COPY scripts/docker /scripts/docker +# Only copy install_mysql.sh to not invalidate cache on other script changes +COPY scripts/docker/install_mysql.sh /scripts/docker/install_mysql.sh # fix permission issue in Azure DevOps when running the script RUN chmod a+x /scripts/docker/install_mysql.sh -RUN ./scripts/docker/install_mysql.sh dev +RUN /scripts/docker/install_mysql.sh dev RUN adduser airflow \ && echo "airflow ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/airflow \ @@ -127,6 +125,7 @@ ARG RUNTIME_APT_DEPS="\ ca-certificates \ software-properties-common \ krb5-user \ + krb5-user \ ldap-utils \ less \ libpython2.7-stdlib \ @@ -151,13 +150,15 @@ ENV RUNTIME_APT_COMMAND=${RUNTIME_APT_COMMAND} ARG ADDITIONAL_RUNTIME_APT_COMMAND="" ENV ADDITIONAL_RUNTIME_APT_COMMAND=${ADDITIONAL_RUNTIME_APT_COMMAND} -ARG ADDITIONAL_RUNTIME_ENV_VARS="" +ARG ADDITIONAL_DEV_APT_ENV="" +ARG ADDITIONAL_RUNTIME_APT_ENV="" # Note missing man directories on debian-buster # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199 RUN mkdir -pv /usr/share/man/man1 \ && mkdir -pv /usr/share/man/man7 \ - && export ${ADDITIONAL_RUNTIME_ENV_VARS?} \ + && export ${ADDITIONAL_DEV_APT_ENV?} \ + && export ${ADDITIONAL_RUNTIME_APT_ENV?} \ && bash -o pipefail -e -u -x -c "${RUNTIME_APT_COMMAND}" \ && bash -o pipefail -e -u -x -c "${ADDITIONAL_RUNTIME_APT_COMMAND}" \ && apt-get update \ @@ -174,12 +175,6 @@ ENV DOCKER_CLI_VERSION=${DOCKER_CLI_VERSION} RUN curl https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_CLI_VERSION}.tgz \ | tar -C /usr/bin --strip-components=1 -xvzf - docker/docker -# Setup PIP -# By default PIP install run without cache to make image smaller -ARG PIP_NO_CACHE_DIR="true" -ENV PIP_NO_CACHE_DIR=${PIP_NO_CACHE_DIR} -RUN echo "Pip no cache dir: ${PIP_NO_CACHE_DIR}" - ARG HOME=/root ENV HOME=${HOME} @@ -195,10 +190,6 @@ RUN mkdir -pv ${AIRFLOW_HOME} && \ mkdir -pv ${AIRFLOW_HOME}/dags && \ mkdir -pv ${AIRFLOW_HOME}/logs -# Increase the value here to force reinstalling Apache Airflow pip dependencies -ARG PIP_DEPENDENCIES_EPOCH_NUMBER="5" -ENV PIP_DEPENDENCIES_EPOCH_NUMBER=${PIP_DEPENDENCIES_EPOCH_NUMBER} - # Install BATS and its dependencies for "in container" tests ARG BATS_VERSION="0.4.0" ARG BATS_SUPPORT_VERSION="0.3.0" @@ -227,7 +218,6 @@ RUN echo "export PATH=/opt/bats/bin:${PATH}" >> /root/.bashrc COPY scripts/docker/load.bash /opt/bats/lib/ RUN chmod a+x /opt/bats/lib/load.bash - # Optimizing installation of Cassandra driver # Speeds up building the image - cassandra driver without CYTHON saves around 10 minutes ARG CASS_DRIVER_NO_CYTHON="1" @@ -272,17 +262,47 @@ ENV INSTALL_FROM_DOCKER_CONTEXT_FILES=${INSTALL_FROM_DOCKER_CONTEXT_FILES} ARG INSTALL_FROM_PYPI="true" ENV INSTALL_FROM_PYPI=${INSTALL_FROM_PYPI} +ARG AIRFLOW_PIP_VERSION=20.2.4 +ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} + +# In the CI image we always: +# * install MySQL +# * install airflow from current sources, not from PyPI package +# * install airflow without `--user` flag +# * install airflow in editable mode +# * install always current version of airflow + +ENV INSTALL_MYSQL_CLIENT="true" +ENV AIRFLOW_INSTALLATION_METHOD="." +ENV AIRFLOW_INSTALL_USER_FLAG="" +ENV AIRFLOW_INSTALL_EDITABLE_FLAG="--editable" +ENV AIRFLOW_INSTALL_VERSION="" + +# Setup PIP +# By default PIP install run without cache to make image smaller +ARG PIP_NO_CACHE_DIR="true" +ENV PIP_NO_CACHE_DIR=${PIP_NO_CACHE_DIR} +RUN echo "Pip no cache dir: ${PIP_NO_CACHE_DIR}" + RUN pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}" +# Increase the value here to force reinstalling Apache Airflow pip dependencies +ARG PIP_DEPENDENCIES_EPOCH_NUMBER="5" +ENV PIP_DEPENDENCIES_EPOCH_NUMBER=${PIP_DEPENDENCIES_EPOCH_NUMBER} + +# Only copy install_airflow_from_latest_master.sh to not invalidate cache on other script changes +COPY scripts/docker/install_airflow_from_latest_master.sh /scripts/docker/install_airflow_from_latest_master.sh +# fix permission issue in Azure DevOps when running the script +RUN chmod a+x /scripts/docker/install_airflow_from_latest_master.sh + # In case of CI builds we want to pre-install master version of airflow dependencies so that # We do not have to always reinstall it from the scratch. # This can be reinstalled from latest master by increasing PIP_DEPENDENCIES_EPOCH_NUMBER. # And is automatically reinstalled from the scratch every time patch release of python gets released +# The Airflow (and providers in case INSTALL_PROVIDERS_FROM_SOURCES is "false") +# are uninstalled, only dependencies remain RUN if [[ ${AIRFLOW_PRE_CACHED_PIP_PACKAGES} == "true" ]]; then \ - pip install \ - "https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]" \ - --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"; \ - pip uninstall --yes apache-airflow; \ + /scripts/docker/install_airflow_from_latest_master.sh; \ fi # Generate random hex dump file so that we can determine whether it's faster to rebuild the image @@ -314,6 +334,23 @@ COPY airflow/__init__.py ${AIRFLOW_SOURCES}/airflow/__init__.py ARG UPGRADE_TO_NEWER_DEPENDENCIES="false" ENV UPGRADE_TO_NEWER_DEPENDENCIES=${UPGRADE_TO_NEWER_DEPENDENCIES} +# Those are additional constraints that are needed for some extras but we do not want to +# Force them on the main Airflow package. Those limitations are: +# * chardet,<4: required by snowflake provider https://github.com/snowflakedb/snowflake-connector-python/blob/v2.3.6/setup.py#L201 +# * lazy-object-proxy<1.5.0: required by astroid +# * pyOpenSSL: Imposed by snowflake provider https://github.com/snowflakedb/snowflake-connector-python/blob/v2.3.6/setup.py#L201 +# * requests>=2.20.0,<2.24.0: required by snowflake provider https://github.com/snowflakedb/snowflake-connector-python/blob/v2.3.6/setup.py#L201 +# * urllib3<1.26: Required to keep boto3 and snowflake happy + +ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="chardet<4 lazy-object-proxy<1.5.0 pyOpenSSL<20.0.0 requests>=2.20.0,<2.24.0 urllib3<1.26" + +ARG CONTINUE_ON_PIP_CHECK_FAILURE="false" + +# Only copy install_airflow.sh to not invalidate cache on other script changes +COPY scripts/docker/install_airflow.sh /scripts/docker/install_airflow.sh +# fix permission issue in Azure DevOps when running the script +RUN chmod a+x /scripts/docker/install_airflow.sh + # The goal of this line is to install the dependencies from the most current setup.py from sources # This will be usually incremental small set of packages in CI optimized build, so it will be very fast # In non-CI optimized build this will install all dependencies before installing sources. @@ -321,49 +358,20 @@ ENV UPGRADE_TO_NEWER_DEPENDENCIES=${UPGRADE_TO_NEWER_DEPENDENCIES} # But in cron job we will install latest versions matching setup.py to see if there is no breaking change # and push the constraints if everything is successful RUN if [[ ${INSTALL_FROM_PYPI} == "true" ]]; then \ - if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then \ - pip install -e ".[${AIRFLOW_EXTRAS}]" --upgrade --upgrade-strategy eager; \ - pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}"; \ - else \ - pip install -e ".[${AIRFLOW_EXTRAS}]" --upgrade --upgrade-strategy only-if-needed\ - --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"; \ - pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}"; \ - fi; \ + /scripts/docker/install_airflow.sh; \ fi +# Only copy install_from_docker_context_files.sh to not invalidate cache on other script changes +COPY scripts/docker/install_from_docker_context_files.sh /scripts/docker/install_from_docker_context_files.sh +# fix permission issue in Azure DevOps when running the script +RUN chmod a+x /scripts/docker/install_from_docker_context_files.sh + # If wheel files are found in /docker-context-files during installation # they are also installed additionally to whatever is installed from Airflow. COPY docker-context-files/ /docker-context-files/ -# hadolint ignore=SC2086, SC2010 RUN if [[ ${INSTALL_FROM_DOCKER_CONTEXT_FILES} == "true" ]]; then \ - reinstalling_apache_airflow_package=$(ls /docker-context-files/apache?airflow?[0-9]*.{whl,tar.gz} 2>/dev/null || true); \ - if [[ "${reinstalling_apache_airflow_package}" != "" ]]; then \ - # install airflow with extras \ - reinstalling_apache_airflow_package="${reinstalling_apache_airflow_package}[${AIRFLOW_EXTRAS}]"; \ - fi; \ - reinstalling_apache_airflow_providers_packages=$(ls /docker-context-files/apache?airflow?providers*.{whl,tar.gz} 2>/dev/null || true); \ - if [[ ${reinstalling_apache_airflow_package} != "" || \ - ${reinstalling_apache_airflow_providers_packages} == "" ]]; then \ - if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then \ - pip install --force-reinstall --upgrade --upgrade-strategy eager \ - --user ${reinstalling_apache_airflow_package} \ - ${reinstalling_apache_airflow_providers_packages}; \ - pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}"; \ - else \ - pip install --force-reinstall --upgrade --upgrade-strategy only-if-needed \ - --user ${reinstalling_apache_airflow_package} \ - ${reinstalling_apache_airflow_providers_packages} \ - --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"; \ - pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}"; \ - fi; \ - fi ; \ - # All the other packages we want to reinstall as-is, without dependencies \ - reinstalling_other_packages=$(ls /docker-context-files/*.{whl,tar.gz} 2>/dev/null | \ - grep -v apache_airflow | grep -v apache-airflow || true); \ - if [[ "${reinstalling_other_packages}" != "" ]]; then \ - pip install --force-reinstall --user --no-deps ${reinstalling_other_packages}; \ - fi; \ + /scripts/docker/install_from_docker_context_files.sh; \ fi # Copy all the www/ files we need to compile assets. Done as two separate COPY diff --git a/IMAGES.rst b/IMAGES.rst index 5e14680095..cd72a0606d 100644 --- a/IMAGES.rst +++ b/IMAGES.rst @@ -473,6 +473,16 @@ The following build arguments (``--build-arg`` in docker build command) can be u +------------------------------------------+------------------------------------------+------------------------------------------+ | ``AIRFLOW_EXTRAS`` | ``all`` | extras to install | +------------------------------------------+------------------------------------------+------------------------------------------+ +| ``UPGRADE_TO_NEWER_DEPENDENCIES`` | ``false`` | If set to true, the dependencies are | +| | | upgraded to newer versions matching | +| | | setup.py before installation. | ++------------------------------------------+------------------------------------------+------------------------------------------+ +| ``CONTINUE_ON_PIP_CHECK_FAILURE`` | ``false`` | By default the image will fail if pip | +| | | check fails for it. This is good for | +| | | interactive building but on CI the | +| | | image should be built regardless - we | +| | | have a separate step to verify image. | ++------------------------------------------+------------------------------------------+------------------------------------------+ | ``INSTALL_FROM_PYPI`` | ``true`` | If set to true, Airflow is installed | | | | from pypi. If you want to install | | | | Airflow from externally provided binary | diff --git a/airflow/providers_manager.py b/airflow/providers_manager.py index 3769fb5114..b3c7e5c92d 100644 --- a/airflow/providers_manager.py +++ b/airflow/providers_manager.py @@ -34,7 +34,7 @@ try: import importlib.resources as importlib_resources except ImportError: # Try back-ported to PY<37 `importlib_resources`. - import importlib_resources + import importlib_resources # noqa log = logging.getLogger(__name__) @@ -156,7 +156,7 @@ class ProvidersManager: f"{provider_info_package_name} do not match. Please make sure they are aligned" ) if package_name not in self._provider_dict: - self._provider_dict[package_name] = (version, provider_info) + self._provider_dict[package_name] = ProviderInfo(version, provider_info) else: log.warning( "The provider for package '%s' could not be registered from because providers for that " @@ -220,7 +220,7 @@ class ProvidersManager: version = provider_info['versions'][0] if package_name not in self._provider_dict: - self._provider_dict[package_name] = (version, provider_info) + self._provider_dict[package_name] = ProviderInfo(version, provider_info) else: log.warning( "The providers for package '%s' could not be registered because providers for that " diff --git a/breeze b/breeze index d11fa828c9..e2eb865882 100755 --- a/breeze +++ b/breeze @@ -125,6 +125,17 @@ function breeze::setup_default_breeze_constants() { # If set to true, RBAC UI will not be used for 1.10 version export DISABLE_RBAC="false" + # Sources by default are installed from local sources when using breeze + AIRFLOW_SOURCES_FROM=${AIRFLOW_SOURCES_FROM:="."} + export AIRFLOW_SOURCES_FROM + + # They are copied to /opt/airflow by default in breeze + AIRFLOW_SOURCES_TO=${AIRFLOW_SOURCES_TO:="/opt/airflow"} + export AIRFLOW_SOURCES_TO + + # Unlike in CI scripts, in breeze by default production image ist installed from sources + export AIRFLOW_INSTALLATION_METHOD="." + # If it set is set to specified version, then the source version of Airflow # is removed and the specified version of Airflow is installed from PyPi export INSTALL_AIRFLOW_VERSION="" @@ -514,6 +525,10 @@ EOF EOF fi fi + if [[ ${VERBOSE} == "true" ]]; then + initialization::summarize_build_environment + fi + } ####################################################################################################### @@ -1154,12 +1169,24 @@ function breeze::parse_arguments() { echo shift ;; + --continue-on-pip-check-failure) + export CONTINUE_ON_PIP_CHECK_FAILURE="true" + echo "Skip PIP check failure." + echo + shift + ;; --package-format) export PACKAGE_FORMAT="${2}" echo "Selected package type: ${PACKAGE_FORMAT}" echo shift 2 ;; + --installation-method) + export AIRFLOW_INSTALLATION_METHOD="${2}" + echo "Airflow installation method: ${PACKAGE_FORMAT}" + echo + shift 2 + ;; --test-type) export TEST_TYPE="${2}" echo "Selected test type: ${TEST_TYPE}" @@ -1465,6 +1492,10 @@ function breeze::prepare_formatted_versions() { tr '\n' ' ' | fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/") readonly FORMATTED_INSTALL_AIRFLOW_VERSIONS + FORMATTED_INSTALLATION_METHOD=$(echo "${_breeze_allowed_installation_methods=""}" | + tr '\n' ' ' | fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/") + readonly FORMATTED_INSTALLATION_METHOD + FORMATTED_POSTGRES_VERSIONS=$(echo "${_breeze_allowed_postgres_versions=""}" | tr '\n' ' ' | fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/") readonly FORMATTED_POSTGRES_VERSIONS @@ -2290,6 +2321,12 @@ ${FORMATTED_INSTALL_AIRFLOW_VERSIONS} image building time in production image and at container entering time for CI image. This can be a GitHub branch like master or v1-10-test, or a tag like 2.0.0a1. +--installation-method INSTALLATION_METHOD + Method of installing airflow - either from the sources ('.') or from package + 'apache-airflow' to install from PyPI. Default in Breeze is to install from sources. One of: + +${FORMATTED_INSTALLATION_METHOD} + --no-rbac-ui Disables RBAC UI when Airflow 1.10.* is installed. @@ -2301,6 +2338,9 @@ ${FORMATTED_INSTALL_AIRFLOW_VERSIONS} --upgrade-to-newer-dependencies Upgrades PIP packages to latest versions available without looking at the constraints. +--continue-on-pip-check-failure + Continue even if 'pip check' fails. + " } diff --git a/breeze-complete b/breeze-complete index 723d4ed1a7..6f06edecdf 100644 --- a/breeze-complete +++ b/breeze-complete @@ -35,6 +35,7 @@ _breeze_allowed_postgres_versions="9.6 10 11 12 13" _breeze_allowed_kind_operations="start stop restart status deploy test shell k9s" _breeze_allowed_test_types="All Core Providers API CLI Integration Other WWW Heisentests Postgres MySQL Helm" _breeze_allowed_package_formats="wheel sdist both" +_breeze_allowed_installation_methods=". apache-airflow" # shellcheck disable=SC2034 { @@ -168,7 +169,7 @@ additional-extras: additional-python-deps: disable-pypi-when-building skip-insta dev-apt-deps: additional-dev-apt-deps: dev-apt-command: additional-dev-apt-command: additional-dev-apt-env: runtime-apt-deps: additional-runtime-apt-deps: runtime-apt-command: additional-runtime-apt-command: additional-runtime-apt-env: load-default-connections load-example-dags -install-packages-from-dist no-rbac-ui package-format: upgrade-to-newer-dependencies +install-packages-from-dist no-rbac-ui package-format: upgrade-to-newer-dependencies installation-method: continue-on-pip-check-failure test-type: preserve-volumes " @@ -291,6 +292,9 @@ function breeze_complete::get_known_values_breeze() { --package-format) _breeze_known_values="${_breeze_allowed_package_formats}" ;; + --installation-method) + _breeze_known_values="${_breeze_allowed_installation_methods}" + ;; *) _breeze_known_values="" ;; diff --git a/docs/apache-airflow/production-deployment.rst b/docs/apache-airflow/production-deployment.rst index 881ee39357..78903b2736 100644 --- a/docs/apache-airflow/production-deployment.rst +++ b/docs/apache-airflow/production-deployment.rst @@ -568,6 +568,12 @@ The following build arguments (``--build-arg`` in docker build command) can be u | | | upgraded to newer versions matching | | | | setup.py before installation. | +------------------------------------------+------------------------------------------+------------------------------------------+ +| ``CONTINUE_ON_PIP_CHECK_FAILURE`` | ``false`` | By default the image build fails if pip | +| | | check fails for it. This is good for | +| | | interactive building but on CI the | +| | | image should be built regardless - we | +| | | have a separate step to verify image. | ++------------------------------------------+------------------------------------------+------------------------------------------+ | ``ADDITIONAL_AIRFLOW_EXTRAS`` | | Optional additional extras with which | | | | airflow is installed. | +------------------------------------------+------------------------------------------+------------------------------------------+ diff --git a/scripts/ci/images/ci_build_dockerhub.sh b/scripts/ci/images/ci_build_dockerhub.sh index decf22432b..217127c59c 100755 --- a/scripts/ci/images/ci_build_dockerhub.sh +++ b/scripts/ci/images/ci_build_dockerhub.sh @@ -112,8 +112,6 @@ else export DOCKER_CACHE="local" # Name the image based on the TAG rather than based on the branch name export FORCE_AIRFLOW_PROD_BASE_TAG="${DOCKER_TAG}" - export AIRFLOW_SOURCES_FROM="empty" - export AIRFLOW_SOURCES_TO="/empty" export INSTALL_AIRFLOW_VERSION="${DOCKER_TAG%-python*}" export AIRFLOW_CONSTRAINTS_REFERENCE="constraints-${INSTALL_AIRFLOW_VERSION}" diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh index 8b637ef8e9..7883726000 100644 --- a/scripts/ci/libraries/_build_images.sh +++ b/scripts/ci/libraries/_build_images.sh @@ -660,6 +660,7 @@ Docker building ${AIRFLOW_CI_IMAGE}. --build-arg INSTALL_FROM_PYPI="${INSTALL_FROM_PYPI}" \ --build-arg INSTALL_FROM_DOCKER_CONTEXT_FILES="${INSTALL_FROM_DOCKER_CONTEXT_FILES}" \ --build-arg UPGRADE_TO_NEWER_DEPENDENCIES="${UPGRADE_TO_NEWER_DEPENDENCIES}" \ + --build-arg CONTINUE_ON_PIP_CHECK_FAILURE="${CONTINUE_ON_PIP_CHECK_FAILURE}" \ --build-arg BUILD_ID="${CI_BUILD_ID}" \ --build-arg COMMIT_SHA="${COMMIT_SHA}" \ "${additional_dev_args[@]}" \ @@ -698,14 +699,19 @@ function build_images::prepare_prod_build() { # When --install-airflow-version is used then the image is build from PIP package EXTRA_DOCKER_PROD_BUILD_FLAGS=( "--build-arg" "AIRFLOW_INSTALLATION_METHOD=apache-airflow" - "--build-arg" "AIRFLOW_INSTALL_VERSION===${INSTALL_AIRFLOW_VERSION}" + "--build-arg" "AIRFLOW_INSTALL_VERSION=${INSTALL_AIRFLOW_VERSION}" "--build-arg" "AIRFLOW_VERSION=${INSTALL_AIRFLOW_VERSION}" ) export AIRFLOW_VERSION="${INSTALL_AIRFLOW_VERSION}" build_images::add_build_args_for_remote_install else - # When no airflow version/reference is specified, production image is built from local sources + # When no airflow version/reference is specified, production image is built either from the + # local sources (in Breeze) or from PyPI (in the ci_scripts) + # Default values for the variables are set in breeze (breeze defaults) and _initialization.sh (CI ones) EXTRA_DOCKER_PROD_BUILD_FLAGS=( + "--build-arg" "AIRFLOW_SOURCES_FROM=${AIRFLOW_SOURCES_FROM}" + "--build-arg" "AIRFLOW_SOURCES_TO=${AIRFLOW_SOURCES_TO}" + "--build-arg" "AIRFLOW_INSTALLATION_METHOD=${AIRFLOW_INSTALLATION_METHOD}" "--build-arg" "AIRFLOW_CONSTRAINTS_REFERENCE=${DEFAULT_CONSTRAINTS_BRANCH}" ) fi @@ -776,6 +782,7 @@ function build_images::build_prod_images() { if [[ ${DEV_APT_COMMAND} != "" ]]; then additional_dev_args+=("--build-arg" "DEV_APT_COMMAND=\"${DEV_APT_COMMAND}\"") fi + docker build \ "${EXTRA_DOCKER_PROD_BUILD_FLAGS[@]}" \ --build-arg PYTHON_BASE_IMAGE="${PYTHON_BASE_IMAGE}" \ @@ -795,6 +802,7 @@ function build_images::build_prod_images() { --build-arg INSTALL_FROM_PYPI="${INSTALL_FROM_PYPI}" \ --build-arg INSTALL_FROM_DOCKER_CONTEXT_FILES="${INSTALL_FROM_DOCKER_CONTEXT_FILES}" \ --build-arg UPGRADE_TO_NEWER_DEPENDENCIES="${UPGRADE_TO_NEWER_DEPENDENCIES}" \ + --build-arg CONTINUE_ON_PIP_CHECK_FAILURE="${CONTINUE_ON_PIP_CHECK_FAILURE}" \ --build-arg BUILD_ID="${CI_BUILD_ID}" \ --build-arg COMMIT_SHA="${COMMIT_SHA}" \ "${DOCKER_CACHE_PROD_BUILD_DIRECTIVE[@]}" \ @@ -826,6 +834,7 @@ function build_images::build_prod_images() { --build-arg INSTALL_FROM_PYPI="${INSTALL_FROM_PYPI}" \ --build-arg INSTALL_FROM_DOCKER_CONTEXT_FILES="${INSTALL_FROM_DOCKER_CONTEXT_FILES}" \ --build-arg UPGRADE_TO_NEWER_DEPENDENCIES="${UPGRADE_TO_NEWER_DEPENDENCIES}" \ + --build-arg CONTINUE_ON_PIP_CHECK_FAILURE="${CONTINUE_ON_PIP_CHECK_FAILURE}" \ --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \ --build-arg AIRFLOW_BRANCH="${AIRFLOW_BRANCH_FOR_PYPI_PRELOADING}" \ --build-arg AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS}" \ @@ -952,7 +961,7 @@ function build_images::build_prod_images_from_locally_built_airflow_packages() { # Useful information for people who stumble upon a pip check failure function build_images::inform_about_pip_check() { - echo """ + echo """ ${COLOR_BLUE}***** Beginning of the instructions ****${COLOR_RESET} The image did not pass 'pip check' verification. This means that there are some conflicting dependencies @@ -964,50 +973,49 @@ It can mean one of those: 2) You changed some dependencies in setup.py or setup.cfg and they are conflicting. + In case 1) - apologies for the trouble.Please let committers know and they will fix it. You might be asked to rebase to the latest master after the problem is fixed. In case 2) - Follow the steps below: -* consult the committers if you are unsure what to do. Just comment in the PR that you need help, if you do, - but try to follow those instructions first! +* try to build CI and then PROD image locally with breeze, adding --upgrade-to-newer-dependencies flag + (repeat it for all python versions) -* ask the committer to set 'upgrade to newer dependencies'. All dependencies in your PR will be updated - to latest 'good' versions and you will be able to check if they are not conflicting. +CI image: -* run locally the image that is failing with Breeze: +${COLOR_BLUE} + ./breeze build-image --upgrade-to-newer-dependencies --python 3.6 +${COLOR_RESET} - ./breeze ${1}--github-image-id ${GITHUB_REGISTRY_PULL_IMAGE_TAG} --backend ${BACKEND="sqlite"} --python ${PYTHON_MAJOR_MINOR_VERSION} +Production image: -* your setup.py and setup.cfg will be mounted to the container. You will be able to iterate with - different setup.py versions. +${COLOR_BLUE} + ./breeze build-image --production-image --upgrade-to-newer-dependencies --python 3.6 +${COLOR_RESET} -* in container your can run 'pipdeptree' to figure out where the dependency conflict comes from. +* If you want to build the image regardless if 'pip check' fails for it, you can add + --continue-on-pip-check-failure flag and enter the image and inspect dependencies. -* Some useful commands that can help yoy to find out dependencies you have: +CI image: - * 'pipdeptree | less' (you can then search through the dependencies with vim-like shortcuts) +${COLOR_BLUE} + ./breeze build-image --upgrade-to-newer-dependencies --python 3.6 --continue-on-pip-check-failure + docker run -it apache/airflow:master-3.6-ci bash +${COLOR_RESET} - * 'pipdeptree > /files/pipdeptree.txt' - this will produce a pipdeptree.txt file in your source - 'files' directory and you can open it in editor of your choice, +Production image: - * 'pipdeptree | grep YOUR_DEPENDENCY' - to see all the requirements your dependency has as specified - by other packages +${COLOR_BLUE} + ./breeze build-image --production-image --upgrade-to-newer-dependencies --python 3.6 --continue-on-pip-check-failure + docker run -it apache/airflow:master-3.6 bash +${COLOR_RESET} -* figure out which dependency limits should be upgraded. Upgrade them in corresponding setup.py extras - and run pip to upgrade your dependencies accordingly: - - pip install '.[all]' --upgrade --upgrade-strategy eager - -* run pip check to figure out if the dependencies have been fixed. It should let you know which dependencies - are conflicting or (hurray!) if there are no conflicts: - - pip check - -* in some, rare, cases, pip will not limit the requirement in case you specify it in extras, you might - need to add such requirement in 'install_requires' section of setup.cfg instead of extras in setup.py. - -* iterate until all such dependency conflicts are fixed. +* You will see error messages there telling which requirements are conflicting and which packages caused the + conflict. Add the limitation that caused the conflict to EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS + variable in Dockerfile.ci. Note that the limitations might be different for Dockerfile.ci and Dockerfile + because not all packages are installed by default in the PROD Dockerfile. So you might find that you + only need to add the limitation to the Dockerfile.ci ${COLOR_BLUE}***** End of the instructions ****${COLOR_RESET} diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh index fb73ab2707..7ff2510183 100644 --- a/scripts/ci/libraries/_initialization.sh +++ b/scripts/ci/libraries/_initialization.sh @@ -148,55 +148,6 @@ function initialization::initialize_base_variables() { # If no Airflow Home defined - fallback to ${HOME}/airflow AIRFLOW_HOME_DIR=${AIRFLOW_HOME:=${HOME}/airflow} export AIRFLOW_HOME_DIR - - INSTALL_PROVIDERS_FROM_SOURCES=${INSTALL_PROVIDERS_FROM_SOURCES:="true"} - export INSTALL_PROVIDERS_FROM_SOURCES - - INSTALLED_PROVIDERS+=( - "amazon" - "celery" - "cncf.kubernetes" - "docker" - "elasticsearch" - "ftp" - "grpc" - "hashicorp" - "http" - "imap" - "google" - "microsoft.azure" - "mysql" - "postgres" - "redis" - "sendgrid" - "sqlite" - "sftp" - "slack" - "sqlite" - "ssh" - ) - export INSTALLED_PROVIDERS - export INSTALLED_EXTRAS="async,amazon,celery,cncf.kubernetes,docker,dask,elasticsearch,ftp,grpc,hashicorp,http,imap,ldap,google,microsoft.azure,mysql,postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv" - - # default version of PIP USED (This has to be < 20.3 until https://github.com/apache/airflow/issues/12838 is solved) - AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION:="20.2.4"} - export AIRFLOW_PIP_VERSION - - # We also pin version of wheel used to get consistent builds - WHEEL_VERSION=${WHEEL_VERSION:="0.36.1"} - export WHEEL_VERSION - - # Sources by default are installed from local sources when using breeze/ci - AIRFLOW_SOURCES_FROM=${AIRFLOW_SOURCES_FROM:="."} - export AIRFLOW_SOURCES_FROM - - # They are copied to /opt/airflow by default (breeze and ci) - AIRFLOW_SOURCES_TO=${AIRFLOW_SOURCES_TO:="/opt/airflow"} - export AIRFLOW_SOURCES_TO - - # And installed from there (breeze and ci) - AIRFLOW_INSTALL_VERSION=${AIRFLOW_INSTALL_VERSION:="."} - export AIRFLOW_INSTALL_VERSION } # Determine current branch @@ -238,6 +189,12 @@ function initialization::initialize_files_for_rebuild_check() { "setup.cfg" "Dockerfile.ci" ".dockerignore" + "scripts/docker/compile_www_assets.sh" + "scripts/docker/install_additional_dependencies.sh" + "scripts/docker/install_airflow.sh" + "scripts/docker/install_airflow_from_latest_master.sh" + "scripts/docker/install_from_docker_context_files.sh" + "scripts/docker/install_mysql.sh" "airflow/www/package.json" "airflow/www/yarn.lock" "airflow/www/webpack.config.js" @@ -354,12 +311,6 @@ function initialization::initialize_image_build_variables() { awk 'BEGIN { FS="=" } { print $2 }' | tr -d '"') export DEFAULT_PROD_EXTRAS - # Installs different airflow version than current from the sources - export INSTALL_AIRFLOW_VERSION=${INSTALL_AIRFLOW_VERSION:=""} - - # Determines if airflow should be installed from a specified reference in GitHub - export INSTALL_AIRFLOW_REFERENCE=${INSTALL_AIRFLOW_REFERENCE:=""} - # By default we are not upgrading to latest version of constraints when building Docker CI image # This will only be done in cron jobs export UPGRADE_TO_NEWER_DEPENDENCIES=${UPGRADE_TO_NEWER_DEPENDENCIES:="false"} @@ -386,8 +337,8 @@ function initialization::initialize_image_build_variables() { export ADDITIONAL_DEV_APT_ENV="${ADDITIONAL_DEV_APT_ENV:=""}" # Use default RUNTIME_APT_COMMAND export RUNTIME_APT_COMMAND="" - # Use default RUNTIME_APT_DEVS - export RUNTIME_APT_DEVS="" + # Use default RUNTIME_APT_DEPS + export RUNTIME_APT_DEPS="" # Use empty ADDITIONAL_RUNTIME_APT_COMMAND export ADDITIONAL_RUNTIME_APT_COMMAND="" # additional runtime apt dependencies on top of the default ones @@ -402,14 +353,77 @@ function initialization::initialize_image_build_variables() { # additional tag for the image export IMAGE_TAG=${IMAGE_TAG:=""} + INSTALL_PROVIDERS_FROM_SOURCES=${INSTALL_PROVIDERS_FROM_SOURCES:="true"} + export INSTALL_PROVIDERS_FROM_SOURCES + + INSTALLED_PROVIDERS+=( + "amazon" + "celery" + "cncf.kubernetes" + "docker" + "elasticsearch" + "ftp" + "grpc" + "hashicorp" + "http" + "imap" + "google" + "microsoft.azure" + "mysql" + "postgres" + "redis" + "sendgrid" + "sqlite" + "sftp" + "slack" + "sqlite" + "ssh" + ) + export INSTALLED_PROVIDERS + export INSTALLED_EXTRAS="async,amazon,celery,cncf.kubernetes,docker,dask,elasticsearch,ftp,grpc,hashicorp,http,imap,ldap,google,microsoft.azure,mysql,postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv" + + # default version of PIP USED (This has to be < 20.3 until https://github.com/apache/airflow/issues/12838 is solved) + AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION:="20.2.4"} + export AIRFLOW_PIP_VERSION + + # We also pin version of wheel used to get consistent builds + WHEEL_VERSION=${WHEEL_VERSION:="0.36.1"} + export WHEEL_VERSION + + # And installed from there (breeze and ci) + AIRFLOW_INSTALL_VERSION=${AIRFLOW_INSTALL_VERSION:="."} + export AIRFLOW_INSTALL_VERSION + + # By default no sources are copied to image + AIRFLOW_SOURCES_FROM=${AIRFLOW_SOURCES_FROM:="empty"} + export AIRFLOW_SOURCES_FROM + + AIRFLOW_SOURCES_TO=${AIRFLOW_SOURCES_TO:="/empty"} + export AIRFLOW_SOURCES_TO + + # By default in scripts production docker image is installed from PyPI package + export AIRFLOW_INSTALLATION_METHOD=${AIRFLOW_INSTALLATION_METHOD:="apache-airflow"} + + # Installs different airflow version than current from the sources + export INSTALL_AIRFLOW_VERSION=${INSTALL_AIRFLOW_VERSION:=""} + + # Continue on PIP CHECK failure + export CONTINUE_ON_PIP_CHECK_FAILURE=${CONTINUE_ON_PIP_CHECK_FAILURE:="false"} + + # Determines if airflow should be installed from a specified reference in GitHub + export INSTALL_AIRFLOW_REFERENCE=${INSTALL_AIRFLOW_REFERENCE:=""} + # whether installation of Airflow should be done via PIP. You can set it to false if you have # all the binary packages (including airflow) in the docker-context-files folder and use # INSTALL_FROM_DOCKER_CONTEXT_FILES="true" to install it from there. export INSTALL_FROM_PYPI="${INSTALL_FROM_PYPI:="true"}" + # whether installation should be performed from the local wheel packages in "docker-context-files" folder export INSTALL_FROM_DOCKER_CONTEXT_FILES="${INSTALL_FROM_DOCKER_CONTEXT_FILES:="false"}" + # reference to CONSTRAINTS. they can be overwritten manually or replaced with AIRFLOW_CONSTRAINTS_LOCATION export AIRFLOW_CONSTRAINTS_REFERENCE="${AIRFLOW_CONSTRAINTS_REFERENCE:=""}" + # direct constraints Location - can be URL or path to local file. If empty, it will be calculated # based on which Airflow version is installed and from where export AIRFLOW_CONSTRAINTS_LOCATION="${AIRFLOW_CONSTRAINTS_LOCATION:=""}" @@ -558,7 +572,7 @@ function initialization::set_default_python_version_if_empty() { } -function initialization::summarize_ci_environment() { +function initialization::summarize_build_environment() { cat < "${md5sum_file}" + rm -rf "${www_dir}/node_modules" + rm -vf "${www_dir}"/{package.json,yarn.lock,.eslintignore,.eslintrc,.stylelintignore,.stylelintrc,compile_assets.sh,webpack.config.js} + popd || exit 1 + fi +} + +compile_www_assets diff --git a/scripts/docker/install_additional_dependencies.sh b/scripts/docker/install_additional_dependencies.sh new file mode 100755 index 0000000000..9c77da042d --- /dev/null +++ b/scripts/docker/install_additional_dependencies.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# shellcheck disable=SC2086 +set -euo pipefail + +test -v UPGRADE_TO_NEWER_DEPENDENCIES +test -v ADDITIONAL_PYTHON_DEPS +test -v EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS +test -v AIRFLOW_INSTALL_USER_FLAG +test -v AIRFLOW_PIP_VERSION +test -v CONTINUE_ON_PIP_CHECK_FAILURE + +# Installs additional dependencies passed as Argument to the Docker build command +function install_additional_dependencies() { + if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then + echo + echo Installing additional dependencies while upgrading to newer dependencies + echo + pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade --upgrade-strategy eager \ + ${ADDITIONAL_PYTHON_DEPS} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} + # make sure correct PIP version is used + pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" + pip check || ${CONTINUE_ON_PIP_CHECK_FAILURE} + else + echo + echo Installing additional dependencies upgrading only if needed + echo + pip install ${AIRFLOW_INSTALL_USER_FLAG} \ + --upgrade --upgrade-strategy only-if-needed \ + ${ADDITIONAL_PYTHON_DEPS} + # make sure correct PIP version is used + pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" + pip check || ${CONTINUE_ON_PIP_CHECK_FAILURE} + fi +} + +install_additional_dependencies diff --git a/scripts/docker/install_airflow.sh b/scripts/docker/install_airflow.sh new file mode 100755 index 0000000000..e00ba56a0b --- /dev/null +++ b/scripts/docker/install_airflow.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Install airflow using regular 'pip install' command. This install airflow depending on the arguments: +# AIRFLOW_INSTALLATION_METHOD - determines where to install airflow form: +# "." - installs airflow from local sources +# "apache-airflow" - installs airflow from PyPI 'apache-airflow' package +# AIRFLOW_INSTALL_VERSION - optionally specify version to install +# UPGRADE_TO_NEWER_DEPENDENCIES - determines whether eager-upgrade should be performed with the +# dependencies (with EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS added) +# +# shellcheck disable=SC2086 +set -euo pipefail + +test -v AIRFLOW_INSTALLATION_METHOD +test -v AIRFLOW_INSTALL_EDITABLE_FLAG +test -v AIRFLOW_INSTALL_USER_FLAG +test -v INSTALL_MYSQL_CLIENT +test -v UPGRADE_TO_NEWER_DEPENDENCIES +test -v CONTINUE_ON_PIP_CHECK_FAILURE +test -v AIRFLOW_CONSTRAINTS_LOCATION + +function install_airflow() { + # Sanity check for editable installation mode. + if [[ ${AIRFLOW_INSTALLATION_METHOD} != "." && \ + ${AIRFLOW_INSTALL_EDITABLE_FLAG} == "--editable" ]]; then + echo + echo "ERROR! You can only use --editable flag when installing airflow from sources!" + echo " Current installation method is '${AIRFLOW_INSTALLATION_METHOD} and should be '.'" + exit 1 + fi + # Remove mysql from extras if client is not going to be installed + if [[ ${INSTALL_MYSQL_CLIENT} != "true" ]]; then + AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/mysql,} + fi + if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then + echo + echo Installing all packages with eager upgrade + echo + # eager upgrade + pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade --upgrade-strategy eager \ + "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_INSTALL_VERSION}" \ + ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} + # make sure correct PIP version is used + pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" + pip check || ${CONTINUE_ON_PIP_CHECK_FAILURE} + else \ + echo + echo Installing all packages with constraints and upgrade if needed + echo + pip install ${AIRFLOW_INSTALL_USER_FLAG} ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ + "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_INSTALL_VERSION}" \ + --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" + # make sure correct PIP version is used + pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" + # then upgrade if needed without using constraints to account for new limits in setup.py + pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade --upgrade-strategy only-if-needed \ + ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ + "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_INSTALL_VERSION}" \ + # make sure correct PIP version is used + pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" + pip check || ${CONTINUE_ON_PIP_CHECK_FAILURE} + fi +} + +install_airflow diff --git a/scripts/docker/install_airflow_from_latest_master.sh b/scripts/docker/install_airflow_from_latest_master.sh new file mode 100755 index 0000000000..ca7b315f73 --- /dev/null +++ b/scripts/docker/install_airflow_from_latest_master.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# shellcheck disable=SC2086 + +# Installs Airflow from latest master. This is pure optimisation. It is done because we do not want +# to reinstall all dependencies from scratch when setup.py changes. Problem with Docker caching is that +# when a file is changed, when added to docker context, it invalidates the cache and it causes Docker +# build to reinstall all dependencies from scratch. This can take a loooooot of time. Therefore we install +# the dependencies first from master (and uninstall airflow right after) so that we can start installing +# deps from those pre-installed dependencies. It saves few minutes of build time when setup.py changes. +# +# If INSTALL_MYSQL_CLIENT is set to false, mysql extra is removed +# +set -euo pipefail + +test -v INSTALL_MYSQL_CLIENT +test -v AIRFLOW_INSTALL_USER_FLAG +test -v AIRFLOW_REPO +test -v AIRFLOW_BRANCH +test -v AIRFLOW_CONSTRAINTS_LOCATION +test -v AIRFLOW_PIP_VERSION + +function install_airflow_from_latest_master() { + echo + echo Installing airflow from latest master. It is used to cache dependencies + echo + if [[ ${INSTALL_MYSQL_CLIENT} != "true" ]]; then + AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/mysql,} + fi + # Install latest master set of dependencies using constraints \ + pip install ${AIRFLOW_INSTALL_USER_FLAG} \ + "https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]" \ + --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" + # make sure correct PIP version is used + pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" + pip freeze | grep apache-airflow-providers | xargs pip uninstall --yes || true + echo + echo Uninstalling just airflow. Dependencies remain. + echo + pip uninstall --yes apache-airflow +} + +install_airflow_from_latest_master diff --git a/scripts/docker/install_from_docker_context_files.sh b/scripts/docker/install_from_docker_context_files.sh new file mode 100755 index 0000000000..c6042ea95a --- /dev/null +++ b/scripts/docker/install_from_docker_context_files.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# shellcheck disable=SC2086 + +# Installs airflow and provider packages from locally present docker context files +# This is used in CI to install airflow and provider packages in the CI system of ours +# The packages are prepared from current sources and placed in the 'docker-context-files folder +# Then both airflow and provider packages are installed using those packages rather than +# PyPI +set -euo pipefail + +test -v AIRFLOW_EXTRAS +test -v AIRFLOW_INSTALL_USER_FLAG +test -v AIRFLOW_CONSTRAINTS_LOCATION +test -v AIRFLOW_PIP_VERSION +test -v CONTINUE_ON_PIP_CHECK_FAILURE +test -v EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS +test -v UPGRADE_TO_NEWER_DEPENDENCIES + + +function install_airflow_and_providers_from_docker_context_files(){ + # Find Apache Airflow packages in docker-context files + local reinstalling_apache_airflow_package + reinstalling_apache_airflow_package=$(ls \ + /docker-context-files/apache?airflow?[0-9]*.{whl,tar.gz} 2>/dev/null || true) + # Add extras when installing airflow + if [[ "${reinstalling_apache_airflow_package}" != "" ]]; then + reinstalling_apache_airflow_package="${reinstalling_apache_airflow_package}[${AIRFLOW_EXTRAS}]" + fi + + # Find Apache Airflow packages in docker-context files + local reinstalling_apache_airflow_providers_packages + reinstalling_apache_airflow_providers_packages=$(ls \ + /docker-context-files/apache?airflow?providers*.{whl,tar.gz} 2>/dev/null || true) + if [[ ${reinstalling_apache_airflow_package} == "" && \ + ${reinstalling_apache_airflow_providers_packages} == "" ]]; then + return + fi + + if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then + echo + echo Force re-installing airflow and providers from local files with eager upgrade + echo + # force reinstall all airflow + provider package local files with eager upgrade + pip install ${AIRFLOW_INSTALL_USER_FLAG} --force-reinstall --upgrade --upgrade-strategy eager \ + ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages} \ + ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} + # make sure correct PIP version is used + pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" + pip check || ${CONTINUE_ON_PIP_CHECK_FAILURE} + else + echo + echo Force re-installing airflow and providers from local files with constraints and upgrade if needed + echo + # force reinstall airflow + provider package local files with constraints + upgrade if needed + pip install ${AIRFLOW_INSTALL_USER_FLAG} --force-reinstall \ + ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages} \ + --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" + # make sure correct PIP version is used \ + pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" + # then upgrade if needed without using constraints to account for new limits in setup.py + pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade --upgrade-strategy only-if-needed \ + ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages} + # make sure correct PIP version is used + pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" + pip check || ${CONTINUE_ON_PIP_CHECK_FAILURE} + fi + +} + +# Simply install all other (non-apache-airflow) packages placed in docker-context files +# without dependencies. This is extremely useful in case you want to install via pip-download +# method on air-gaped system where you do not want to download any dependencies from remote hosts +# which is a requirement for serious installations +install_all_other_packages_from_docker_context_files() { + echo + echo Force re-installing all other package from local files without dependencies + echo + local reinstalling_other_packages + # shellcheck disable=SC2010 + reinstalling_other_packages=$(ls /docker-context-files/*.{whl,tar.gz} 2>/dev/null | \ + grep -v apache_airflow | grep -v apache-airflow || true) + if [[ "${reinstalling_other_packages}" != "" ]]; then \ + pip install ${AIRFLOW_INSTALL_USER_FLAG} --force-reinstall --no-deps --no-index ${reinstalling_other_packages} + # make sure correct PIP version is used + pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" + fi +} + +install_airflow_and_providers_from_docker_context_files +install_all_other_packages_from_docker_context_files diff --git a/scripts/docker/install_mysql.sh b/scripts/docker/install_mysql.sh index e152309f66..877e56c845 100755 --- a/scripts/docker/install_mysql.sh +++ b/scripts/docker/install_mysql.sh @@ -15,27 +15,31 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +set -euo pipefail declare -a packages -if [[ "${1}" == "dev" ]]; then - packages=("libmysqlclient-dev" "mysql-client") -elif [[ "${1}" == "prod" ]]; then - packages=("libmysqlclient21" "mysql-client") -else - echo - echo "Specify either prod or dev" - echo -fi +MYSQL_VERSION="8.0" +readonly MYSQL_VERSION -# Install MySQL Client during the container build -set -euo pipefail +install_mysql_client() { + echo + echo Installing mysql client + echo -# Install MySQL client from Oracle repositories (Debian installs mariadb) -# But only if it is not disabled -if [[ ${INSTALL_MYSQL_CLIENT:="true"} == "true" ]]; then - KEY="A4A9406876FCBD3C456770C88C718D3B5072E1F5" - readonly KEY + if [[ "${1}" == "dev" ]]; then + packages=("libmysqlclient-dev" "mysql-client") + elif [[ "${1}" == "prod" ]]; then + packages=("libmysqlclient21" "mysql-client") + else + echo + echo "Specify either prod or dev" + echo + exit 1 + fi + + local key="A4A9406876FCBD3C456770C88C718D3B5072E1F5" + readonly key GNUPGHOME="$(mktemp -d)" export GNUPGHOME @@ -43,16 +47,28 @@ if [[ ${INSTALL_MYSQL_CLIENT:="true"} == "true" ]]; then for keyserver in $(shuf -e ha.pool.sks-keyservers.net hkp://p80.pool.sks-keyservers.net:80 \ keyserver.ubuntu.com hkp://keyserver.ubuntu.com:80) do - gpg --keyserver "${keyserver}" --recv-keys "${KEY}" && break + gpg --keyserver "${keyserver}" --recv-keys "${key}" && break done set -e - gpg --export "${KEY}" > /etc/apt/trusted.gpg.d/mysql.gpg + gpg --export "${key}" > /etc/apt/trusted.gpg.d/mysql.gpg gpgconf --kill all rm -rf "${GNUPGHOME}" + unset GNUPGHOME apt-key list > /dev/null 2>&1 - echo "deb http://repo.mysql.com/apt/debian/ buster mysql-8.0" | tee -a /etc/apt/sources.list.d/mysql.list + echo "deb http://repo.mysql.com/apt/debian/ buster mysql-${MYSQL_VERSION}" | tee -a /etc/apt/sources.list.d/mysql.list apt-get update apt-get install --no-install-recommends -y "${packages[@]}" apt-get autoremove -yqq --purge apt-get clean && rm -rf /var/lib/apt/lists/* +} + + + +# Install MySQL Client during the container build +set -euo pipefail + +# Install MySQL client from Oracle repositories (Debian installs mariadb) +# But only if it is not disabled +if [[ ${INSTALL_MYSQL_CLIENT:="true"} == "true" ]]; then + install_mysql_client "${@}" fi diff --git a/scripts/in_container/entrypoint_ci.sh b/scripts/in_container/entrypoint_ci.sh index 06571c4d6f..d14a364d31 100755 --- a/scripts/in_container/entrypoint_ci.sh +++ b/scripts/in_container/entrypoint_ci.sh @@ -72,6 +72,7 @@ else fi if [[ -z ${INSTALL_AIRFLOW_VERSION=} ]]; then + export PYTHONPATH=${AIRFLOW_SOURCES} echo echo "Using already installed airflow version" echo @@ -81,7 +82,6 @@ if [[ -z ${INSTALL_AIRFLOW_VERSION=} ]]; then sudo rm -rf "${AIRFLOW_SOURCES}"/tmp/* mkdir -p "${AIRFLOW_SOURCES}"/logs/ mkdir -p "${AIRFLOW_SOURCES}"/tmp/ - export PYTHONPATH=${AIRFLOW_SOURCES} elif [[ ${INSTALL_AIRFLOW_VERSION} == "none" ]]; then echo echo "Skip installing airflow - only install wheel/tar.gz packages that are present locally" diff --git a/scripts/in_container/run_mypy.sh b/scripts/in_container/run_mypy.sh index 09845531ba..661c4428a3 100755 --- a/scripts/in_container/run_mypy.sh +++ b/scripts/in_container/run_mypy.sh @@ -20,6 +20,8 @@ . "$( dirname "${BASH_SOURCE[0]}" )/_in_container_script_init.sh" set -x +export PYTHONPATH=${AIRFLOW_SOURCES} + mypy_args=() # Mypy doesn't cope very well with namespace packages when give filenames (it gets confused about the lack of __init__.py in airflow.providers, and thinks airflow.providers.docker is the same as a "docker" top level module). diff --git a/setup.cfg b/setup.cfg index 3d1beab8fb..7f7e7d3037 100644 --- a/setup.cfg +++ b/setup.cfg @@ -104,7 +104,7 @@ install_requires = itsdangerous>=1.1.0 jinja2>=2.10.1, <2.12.0 jsonschema~=3.0 - lazy-object-proxy<1.5.0 # Required to keep pip-check happy with astroid + lazy-object-proxy lockfile>=0.12.2 markdown>=2.5.2, <4.0 markupsafe>=1.1.1, <2.0 @@ -113,11 +113,8 @@ install_requires = pendulum~=2.0 pep562~=1.0;python_version<"3.7" psutil>=4.2.0, <6.0.0 - # snowflake-connector-python don't support newer version - # https://github.com/snowflakedb/snowflake-connector-python/blob/v2.3.6/setup.py#L201 - pyOpenSSL<20.0.0 pygments>=2.0.1, <3.0 - # snowflake-connector-python, flask-jwt-extended, msal are not compatible with newest version. + # Required for flask-jwt-extended and msal pyjwt<2 # python daemon crashes with 'socket operation on non-socket' for python 3.8+ in version < 2.2.4 # https://pagure.io/python-daemon/issue/34 @@ -125,7 +122,7 @@ install_requires = python-dateutil>=2.3, <3 python-nvd3~=0.15.0 python-slugify>=3.0.0,<5.0 - requests>=2.20.0, <2.24.0 # Required to keep snowflake happy + requests>=2.20.0 rich==9.2.0 setproctitle>=1.1.8, <2 sqlalchemy>=1.3.18, <2 @@ -135,7 +132,6 @@ install_requires = termcolor>=1.1.0 typing-extensions>=3.7.4;python_version<"3.8" unicodecsv>=0.14.1 - urllib3<1.26 # Required to keep boto3 happy werkzeug~=1.0, >=1.0.1 [options.packages.find] diff --git a/setup.py b/setup.py index 7cd3925802..1aa73c4ebb 100644 --- a/setup.py +++ b/setup.py @@ -477,7 +477,7 @@ zendesk = [ ############################################################################################################ # IMPORTANT NOTE!!!!!!!!!!!!!!! # IF you are removing dependencies from this list, please make sure that you also increase -# DEPENDENCIES_EPOCH_NUMBER in the Dockerfile.ci +# PIP_DEPENDENCIES_EPOCH_NUMBER in the Dockerfile.ci and Dockerfile ############################################################################################################ devel = [ 'beautifulsoup4~=4.7.1', @@ -526,7 +526,7 @@ devel = [ ############################################################################################################ # IMPORTANT NOTE!!!!!!!!!!!!!!! # If you are removing dependencies from the above list, please make sure that you also increase -# DEPENDENCIES_EPOCH_NUMBER in the Dockerfile.ci +# PIP_DEPENDENCIES_EPOCH_NUMBER in the Dockerfile.ci and Dockerfile ############################################################################################################ devel_minreq = cgroups + devel + doc + kubernetes + mysql + password