Building backport packages is now done inside Breeze container (#8558)

This way you do not have to worry about setting up your environment. Fixes #8537
2020-04-26 12:35:27 +02:00 · 2020-04-26 12:35:27 +02:00 · 3c4ca62af1
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@ -131,20 +131,24 @@ jobs:
        run: ./scripts/ci/ci_build_production_images.sh

  prepare-backport-packages:
-    timeout-minutes: 20
+    timeout-minutes: 60
    name: "Backport packages"
    runs-on: ubuntu-latest
    env:
      INSTALL_AIRFLOW_VERSION: "1.10.10"
      CI_JOB_TYPE: "Prepare & test packages"
+      PYTHON_MAJOR_MINOR_VERSION: 3.6
    steps:
      - uses: actions/checkout@master
      - uses: actions/setup-python@v1
        with:
          python-version: '3.x'
+      - name: "Free space"
+        run: ./scripts/ci/ci_free_space_on_ci.sh
+      - name: "Build CI image ${{ matrix.python-version }}"
+        run: ./scripts/ci/ci_prepare_image_on_ci.sh
      - name: "Prepare & test backport packages"
        run: |
-          python -m pip install bowler sdist wheel
          ./scripts/ci/ci_prepare_and_test_backport_packages.sh

  pyfiles:
--- a/backport_packages/setup_backport_packages.py
+++ b/backport_packages/setup_backport_packages.py
@ -346,7 +346,7 @@ def do_setup_package_providers(provider_package: str, deps: List[str], extras: D
        long_description=get_long_description(provider_package),
        long_description_content_type='text/markdown',
        license='Apache License 2.0',
-        version='0.0.1',
+        version='2020.04.27.rc1',
        packages=found_packages,
        package_data={
            '': ["airflow/providers/cncf/kubernetes/example_dags/*.yaml"],
@ -366,6 +366,13 @@ def do_setup_package_providers(provider_package: str, deps: List[str], extras: D
            'Programming Language :: Python :: 3.7',
            'Topic :: System :: Monitoring',
        ],
+        setup_requires=[
+            'bowler',
+            'docutils',
+            'gitpython',
+            'setuptools',
+            'wheel',
+        ],
        python_requires='>=3.6',
    )

--- a/requirements/requirements-python3.6.txt
+++ b/requirements/requirements-python3.6.txt
@ -11,6 +11,7 @@ Flask-OpenID==1.2.5
 Flask-SQLAlchemy==2.4.1
 Flask-WTF==0.14.3
 Flask==1.1.2
+GitPython==3.1.1
 HeapDict==1.0.1
 JPype1==0.7.3
 JayDeBeApi==1.1.1
@ -68,9 +69,10 @@ beautifulsoup4==4.7.1
 billiard==3.6.3.0
 black==19.10b0
 blinker==1.4
-boto3==1.12.44
+boto3==1.12.46
 boto==2.49.0
-botocore==1.15.44
+botocore==1.15.46
+bowler==0.8.0
 cached-property==1.5.1
 cachetools==4.1.0
 cassandra-driver==3.20.2
@ -89,16 +91,16 @@ colorama==0.4.3
 colorlog==4.0.2
 coverage==5.1
 croniter==0.3.31
-cryptography==2.9.1
+cryptography==2.9.2
 curlify==2.2.1
 cx-Oracle==7.3.0
-dask==2.14.0
+dask==2.15.0
 datadog==0.35.0
 decorator==4.4.2
 defusedxml==0.6.0
 dill==0.3.1.1
 distlib==0.3.0
-distributed==2.14.0
+distributed==2.15.0
 dnspython==1.16.0
 docker-pycreds==0.4.0
 docker==3.7.3
@ -113,6 +115,7 @@ execnet==1.7.1
 facebook-business==6.0.3
 fastavro==0.23.2
 filelock==3.0.12
+fissix==19.2b1
 flake8-colors==0.1.6
 flake8==3.7.9
 flaky==3.6.1
@ -124,6 +127,7 @@ funcsigs==1.0.2
 future-fstrings==1.2.0
 future==0.18.2
 gcsfs==0.6.1
+gitdb==4.0.4
 google-ads==4.0.0
 google-api-core==1.17.0
 google-api-python-client==1.8.2
@ -166,7 +170,7 @@ hmsclient==0.1.1
 httplib2==0.17.3
 humanize==0.5.1
 hvac==0.10.1
-identify==1.4.14
+identify==1.4.15
 idna==2.9
 ijson==2.6.1
 imagesize==1.2.0
@ -225,7 +229,7 @@ oscrypto==1.2.0
 packaging==20.3
 pandas-gbq==0.13.1
 pandas==1.0.3
-papermill==2.1.0
+papermill==2.1.1
 parameterized==0.7.4
 paramiko==2.7.1
 parso==0.7.0
@ -254,7 +258,7 @@ pycodestyle==2.5.0
 pycountry==19.8.18
 pycparser==2.20
 pycryptodomex==3.9.7
-pydata-google-auth==1.0.0
+pydata-google-auth==1.1.0
 pydruid==0.5.8
 pyexasol==0.12.0
 pyflakes==2.1.1
@ -303,9 +307,11 @@ sendgrid==6.2.2
 sentinels==1.0.0
 sentry-sdk==0.14.3
 setproctitle==1.1.10
+sh==1.12.14
 simple-salesforce==1.0.0
 six==1.14.0
 slackclient==1.3.2
+smmap==3.0.2
 snowballstemmer==2.0.0
 snowflake-connector-python==2.2.4
 snowflake-sqlalchemy==1.2.3
--- a/requirements/requirements-python3.7.txt
+++ b/requirements/requirements-python3.7.txt
@ -11,6 +11,7 @@ Flask-OpenID==1.2.5
 Flask-SQLAlchemy==2.4.1
 Flask-WTF==0.14.3
 Flask==1.1.2
+GitPython==3.1.1
 HeapDict==1.0.1
 JPype1==0.7.3
 JayDeBeApi==1.1.1
@ -68,9 +69,10 @@ beautifulsoup4==4.7.1
 billiard==3.6.3.0
 black==19.10b0
 blinker==1.4
-boto3==1.12.44
+boto3==1.12.46
 boto==2.49.0
-botocore==1.15.44
+botocore==1.15.46
+bowler==0.8.0
 cached-property==1.5.1
 cachetools==4.1.0
 cassandra-driver==3.20.2
@ -89,16 +91,16 @@ colorama==0.4.3
 colorlog==4.0.2
 coverage==5.1
 croniter==0.3.31
-cryptography==2.9.1
+cryptography==2.9.2
 curlify==2.2.1
 cx-Oracle==7.3.0
-dask==2.14.0
+dask==2.15.0
 datadog==0.35.0
 decorator==4.4.2
 defusedxml==0.6.0
 dill==0.3.1.1
 distlib==0.3.0
-distributed==2.14.0
+distributed==2.15.0
 dnspython==1.16.0
 docker-pycreds==0.4.0
 docker==3.7.3
@ -113,6 +115,7 @@ execnet==1.7.1
 facebook-business==6.0.3
 fastavro==0.23.2
 filelock==3.0.12
+fissix==19.2b1
 flake8-colors==0.1.6
 flake8==3.7.9
 flaky==3.6.1
@ -124,6 +127,7 @@ funcsigs==1.0.2
 future-fstrings==1.2.0
 future==0.18.2
 gcsfs==0.6.1
+gitdb==4.0.4
 google-ads==5.0.4
 google-api-core==1.17.0
 google-api-python-client==1.8.2
@ -166,7 +170,7 @@ hmsclient==0.1.1
 httplib2==0.17.3
 humanize==0.5.1
 hvac==0.10.1
-identify==1.4.14
+identify==1.4.15
 idna==2.9
 ijson==2.6.1
 imagesize==1.2.0
@ -224,7 +228,7 @@ oscrypto==1.2.0
 packaging==20.3
 pandas-gbq==0.13.1
 pandas==1.0.3
-papermill==2.1.0
+papermill==2.1.1
 parameterized==0.7.4
 paramiko==2.7.1
 parso==0.7.0
@ -252,7 +256,7 @@ pycodestyle==2.5.0
 pycountry==19.8.18
 pycparser==2.20
 pycryptodomex==3.9.7
-pydata-google-auth==1.0.0
+pydata-google-auth==1.1.0
 pydruid==0.5.8
 pyexasol==0.12.0
 pyflakes==2.1.1
@ -301,9 +305,11 @@ sendgrid==6.2.2
 sentinels==1.0.0
 sentry-sdk==0.14.3
 setproctitle==1.1.10
+sh==1.12.14
 simple-salesforce==1.0.0
 six==1.14.0
 slackclient==1.3.2
+smmap==3.0.2
 snowballstemmer==2.0.0
 snowflake-connector-python==2.2.4
 snowflake-sqlalchemy==1.2.3
--- a/requirements/setup-3.6.md5
+++ b/requirements/setup-3.6.md5
@ -1 +1 @@
-90fcd0e330052de1268264634e0d4035  /opt/airflow/setup.py
+62ecf8b76ba00c5d680606d585a7164b  /opt/airflow/setup.py
--- a/requirements/setup-3.7.md5
+++ b/requirements/setup-3.7.md5
@ -1 +1 @@
-90fcd0e330052de1268264634e0d4035  /opt/airflow/setup.py
+62ecf8b76ba00c5d680606d585a7164b  /opt/airflow/setup.py
--- a/scripts/ci/_utils.sh
+++ b/scripts/ci/_utils.sh
@ -1524,6 +1524,25 @@ function run_generate_requirements() {
        | tee -a "${OUTPUT_LOG}"
 }

+function run_prepare_packages() {
+    docker run "${EXTRA_DOCKER_FLAGS[@]}" \
+        --entrypoint "/usr/local/bin/dumb-init"  \
+        --env PYTHONDONTWRITEBYTECODE \
+        --env VERBOSE \
+        --env VERBOSE_COMMANDS \
+        --env HOST_USER_ID="$(id -ur)" \
+        --env HOST_GROUP_ID="$(id -gr)" \
+        --env UPGRADE_WHILE_GENERATING_REQUIREMENTS \
+        --env PYTHON_MAJOR_MINOR_VERSION \
+        --env SHOW_GENERATE_REQUIREMENTS_INSTRUCTIONS \
+        -t \
+        -v "${AIRFLOW_SOURCES}:/opt/airflow" \
+        --rm \
+        "${AIRFLOW_CI_IMAGE}" \
+        "--" "/opt/airflow/scripts/ci/in_container/run_prepare_packages.sh" "${@}" \
+        | tee -a "${OUTPUT_LOG}"
+}
+

 function get_airflow_version_from_production_image() {
     docker run --entrypoint /bin/bash "${AIRFLOW_PROD_IMAGE}" -c 'echo "${AIRFLOW_VERSION}"'
--- a/scripts/ci/ci_prepare_packages.sh
+++ b/scripts/ci/ci_prepare_packages.sh
@ -19,100 +19,11 @@ export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}

 # shellcheck source=scripts/ci/_script_init.sh
 . "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
-LIST_OF_DIRS_FILE=$(mktemp)

-cd "${MY_DIR}/../../airflow/providers" || exit 1
+get_ci_environment

-find . -type d | sed 's/.\///; s/\//\./g' | grep -E 'hooks|operators|sensors|secrets' \
-    > "${LIST_OF_DIRS_FILE}"
+prepare_ci_build

-cd "${MY_DIR}/../../backport_packages" || exit 1
+rebuild_ci_image_if_needed

-rm -rf dist/*
-rm -rf -- *.egg-info
-
-if [[ -z "$*" ]]; then
-    BACKPORT_PACKAGES=$(python3 setup_backport_packages.py list-backport-packages)
-    BUILD_AIRFLOW_PACKAGE="true"
-
-    PACKAGE_ERROR="false"
-    # Check if all providers are included
-    for PACKAGE in ${BACKPORT_PACKAGES}
-    do
-        if ! grep -E "^${PACKAGE}" <"${LIST_OF_DIRS_FILE}" >/dev/null; then
-            echo "The package ${PACKAGE} is not available in providers dir"
-            PACKAGE_ERROR="true"
-        fi
-        sed -i "/^${PACKAGE}.*/d" "${LIST_OF_DIRS_FILE}"
-    done
-
-    if [[ ${PACKAGE_ERROR} == "true" ]]; then
-        echo
-        echo "ERROR! Some packages from backport_packages/setup_backport_packages.py are missing in providers dir"
-        exit 1
-    fi
-
-    NUM_LINES=$(wc -l "${LIST_OF_DIRS_FILE}" | awk '{ print $1 }')
-    if [[ ${NUM_LINES} != "0" ]]; then
-        echo "ERROR! Some folders from providers package are not defined"
-        echo "       Please add them to backport_packages/setup_backport_packages.py:"
-        echo
-        cat "${LIST_OF_DIRS_FILE}"
-        echo
-        exit 1
-    fi
-else
-    if [[ "$1" == "--help" ]]; then
-        echo
-        echo "Builds all backport packages."
-        echo
-        echo "You can provide list of packages to build out of:"
-        echo
-        python3 setup_backport_packages.py list-backport-packages | tr '\n ' ' ' | fold -w 100 -s
-        echo
-        echo
-        exit
-    fi
-    BACKPORT_PACKAGES="$*"
-    BUILD_AIRFLOW_PACKAGE="false"
-fi
-
-echo "==================================================================================="
-echo " Copying sources and doing refactor for backporting"
-echo "==================================================================================="
-python3 setup_backport_packages.py prepare
-
-for BACKPORT_PACKAGE in ${BACKPORT_PACKAGES}
-do
-    echo "==================================================================================="
-    echo " Preparing backporting package ${BACKPORT_PACKAGE}"
-    echo "-----------------------------------------------------------------------------------"
-    python3 setup_backport_packages.py "${BACKPORT_PACKAGE}" clean --all >/dev/null 2>&1
-    python3 setup_backport_packages.py "${BACKPORT_PACKAGE}" sdist bdist_wheel >/dev/null 2>&1
-    echo " Prepared backporting package ${BACKPORT_PACKAGE}"
-done
-
-cd "${MY_DIR}/../../" || exit 1
-
-if [[ ${BUILD_AIRFLOW_PACKAGE} == "true" ]]; then
-    echo "==================================================================================="
-    echo " Preparing apache-airflow package"
-    echo "==================================================================================="
-    python3 setup.py clean --all
-    python3 setup.py sdist bdist_wheel >/dev/null
-fi
-echo "==================================================================================="
-
-AIRFLOW_PACKAGES_TGZ_FILE="/tmp/airflow-packages-$(date +"%Y%m%d-%H%M%S").tar.gz"
-
-tar -cvzf "${AIRFLOW_PACKAGES_TGZ_FILE}" dist/*.whl dist/*.tar.gz
-echo
-echo "Airflow packages are in dist folder and tar-gzipped in ${AIRFLOW_PACKAGES_TGZ_FILE}"
-echo
-if [[ "${CI:=false}" == "true" ]]; then
-    echo
-    echo "Sending all airflow packages to file.io"
-    echo
-    curl -F "file=@${AIRFLOW_PACKAGES_TGZ_FILE}" https://file.io
-    echo
-fi
+run_prepare_packages "$@"
--- a/scripts/ci/in_container/_in_container_utils.sh
+++ b/scripts/ci/in_container/_in_container_utils.sh
@ -91,7 +91,7 @@ function in_container_cleanup_pycache() {
 #
 function in_container_fix_ownership() {
    set +o pipefail
-    sudo find . -user root \
+    sudo find "${AIRFLOW_SOURCES}" -user root \
    | sudo xargs chown -v "${HOST_USER_ID}.${HOST_GROUP_ID}" --no-dereference >/dev/null 2>&1
    set -o pipefail
 }
--- a/scripts/ci/in_container/run_prepare_packages.sh
+++ b/scripts/ci/in_container/run_prepare_packages.sh
@ -0,0 +1,122 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# shellcheck source=scripts/ci/in_container/_in_container_script_init.sh
+. "$( dirname "${BASH_SOURCE[0]}" )/_in_container_script_init.sh"
+
+LIST_OF_DIRS_FILE=$(mktemp)
+
+# adding trap to exiting trap
+HANDLERS="$( trap -p EXIT | cut -f2 -d \' )"
+# shellcheck disable=SC2064
+trap "${HANDLERS}${HANDLERS:+;}in_container_fix_ownership" EXIT
+
+cd "${AIRFLOW_SOURCES}/airflow/providers" || exit 1
+
+find . -type d | sed 's/.\///; s/\//\./g' | grep -E 'hooks|operators|sensors|secrets' \
+    > "${LIST_OF_DIRS_FILE}"
+
+cd "${AIRFLOW_SOURCES}/backport_packages" || exit 1
+
+rm -rf dist/*
+rm -rf -- *.egg-info
+
+if [[ -z "$*" ]]; then
+    BACKPORT_PACKAGES=$(python3 setup_backport_packages.py list-backport-packages)
+    BUILD_AIRFLOW_PACKAGE="true"
+
+    PACKAGE_ERROR="false"
+    # Check if all providers are included
+    for PACKAGE in ${BACKPORT_PACKAGES}
+    do
+        if ! grep -E "^${PACKAGE}" <"${LIST_OF_DIRS_FILE}" >/dev/null; then
+            echo "The package ${PACKAGE} is not available in providers dir"
+            PACKAGE_ERROR="true"
+        fi
+        sed -i "/^${PACKAGE}.*/d" "${LIST_OF_DIRS_FILE}"
+    done
+
+    if [[ ${PACKAGE_ERROR} == "true" ]]; then
+        echo
+        echo "ERROR! Some packages from backport_packages/setup_backport_packages.py are missing in providers dir"
+        exit 1
+    fi
+
+    NUM_LINES=$(wc -l "${LIST_OF_DIRS_FILE}" | awk '{ print $1 }')
+    if [[ ${NUM_LINES} != "0" ]]; then
+        echo "ERROR! Some folders from providers package are not defined"
+        echo "       Please add them to backport_packages/setup_backport_packages.py:"
+        echo
+        cat "${LIST_OF_DIRS_FILE}"
+        echo
+        exit 1
+    fi
+else
+    if [[ "$1" == "--help" ]]; then
+        echo
+        echo "Builds all backport packages."
+        echo
+        echo "You can provide list of packages to build out of:"
+        echo
+        python3 setup_backport_packages.py list-backport-packages | tr '\n ' ' ' | fold -w 100 -s
+        echo
+        echo
+        exit
+    fi
+    BACKPORT_PACKAGES="$*"
+    BUILD_AIRFLOW_PACKAGE="false"
+fi
+
+echo "==================================================================================="
+echo " Copying sources and doing refactor for backporting"
+echo "==================================================================================="
+python3 setup_backport_packages.py prepare
+
+for BACKPORT_PACKAGE in ${BACKPORT_PACKAGES}
+do
+    echo "==================================================================================="
+    echo " Preparing backporting package ${BACKPORT_PACKAGE}"
+    echo "-----------------------------------------------------------------------------------"
+    python3 setup_backport_packages.py "${BACKPORT_PACKAGE}" clean --all >/dev/null 2>&1
+    python3 setup_backport_packages.py "${BACKPORT_PACKAGE}" sdist bdist_wheel >/dev/null 2>&1
+    echo " Prepared backporting package ${BACKPORT_PACKAGE}"
+done
+
+cd "${AIRFLOW_SOURCES}" || exit 1
+
+if [[ ${BUILD_AIRFLOW_PACKAGE} == "true" ]]; then
+    echo "==================================================================================="
+    echo " Preparing apache-airflow package"
+    echo "==================================================================================="
+    python3 setup.py clean --all
+    python3 setup.py sdist bdist_wheel >/dev/null
+fi
+echo "==================================================================================="
+
+AIRFLOW_PACKAGES_TGZ_FILE="/tmp/airflow-packages-$(date +"%Y%m%d-%H%M%S").tar.gz"
+
+tar -cvzf "${AIRFLOW_PACKAGES_TGZ_FILE}" dist/*.whl dist/*.tar.gz
+echo
+echo "Airflow packages are in dist folder and tar-gzipped in ${AIRFLOW_PACKAGES_TGZ_FILE}"
+echo
+if [[ "${CI:=false}" == "true" ]]; then
+    echo
+    echo "Sending all airflow packages to file.io"
+    echo
+    curl -F "file=@${AIRFLOW_PACKAGES_TGZ_FILE}" https://file.io
+    echo
+fi
--- a/setup.py
+++ b/setup.py
@ -427,13 +427,16 @@ all_dbs = (cassandra + cloudant + druid + exasol + hdfs + hive + mongo + mssql +
 ############################################################################################################
 devel = [
    'beautifulsoup4~=4.7.1',
+    'bowler',
    'click==6.7',
    'contextdecorator;python_version<"3.4"',
    'coverage',
+    'docutils',
    'flake8>=3.6.0',
    'flake8-colors',
    'flaky',
    'freezegun',
+    'gitpython',
    'ipdb',
    'jira',
    'mongomock',
@ -452,6 +455,8 @@ devel = [
    'pywinrm',
    'qds-sdk>=1.9.6',
    'requests_mock',
+    'setuptools',
+    'wheel',
    'yamllint',
 ]
 ############################################################################################################
@ -637,8 +642,8 @@ def do_setup():
        install_requires=INSTALL_REQUIREMENTS,
        setup_requires=[
            'bowler',
-            'docutils>=0.14, <0.16'
-            'gitpython>=2.0.2',
+            'docutils',
+            'gitpython',
            'setuptools',
            'wheel',
        ],