From e92d50eda7920bae90933da41d928a7196cc88cf Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Tue, 4 Aug 2020 01:26:59 +0200 Subject: [PATCH] Replace file.io with artifacts (#10137) --- .github/workflows/ci.yml | 38 +++++++++++++++++-- .github/workflows/quarantined.yaml | 6 +++ airflow/cli/commands/info_command.py | 2 +- breeze | 11 ++++++ dev/BACKPORT_PACKAGES.md | 2 +- .../ci_prepare_and_test_backport_packages.sh | 6 --- scripts/ci/docker-compose/base.yml | 6 +++ scripts/ci/docker-compose/ci.yml | 28 ++++++++++++++ scripts/ci/docker-compose/files.yml | 22 +++++++++++ scripts/ci/docker-compose/local-prod.yml | 1 - scripts/ci/docker-compose/local.yml | 8 ---- .../ci/in_container/_in_container_utils.sh | 21 ++-------- scripts/ci/in_container/run_ci_tests.sh | 2 +- .../run_prepare_backport_packages.sh | 9 +---- .../run_prepare_backport_readme.sh | 9 +---- scripts/ci/in_container/run_system_tests.sh | 2 +- scripts/ci/libraries/_initialization.sh | 1 + scripts/ci/libraries/_local_mounts.sh | 1 - scripts/ci/testing/ci_run_airflow_testing.sh | 22 ++++++++--- scripts/ci/tools/ci_clear_tmp.sh | 1 + scripts/ci/tools/ci_fix_ownership.sh | 1 + 21 files changed, 137 insertions(+), 62 deletions(-) create mode 100644 scripts/ci/docker-compose/ci.yml create mode 100644 scripts/ci/docker-compose/files.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3735d74626..c98ba9aeb2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,6 +28,7 @@ on: env: MOUNT_LOCAL_SOURCES: "false" + MOUNT_FILES: "true" FORCE_ANSWER_TO_QUESTIONS: "yes" SKIP_CHECK_REMOTE_IMAGE: "true" SKIP_CI_IMAGE_CHECK: "true" @@ -108,6 +109,20 @@ jobs: run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - name: "Prepare & test backport packages" run: "./scripts/ci/backport_packages/ci_prepare_and_test_backport_packages.sh" + - uses: actions/upload-artifact@v2 + name: Upload packages + if: always() + with: + name: > + airflow-backport-packages + path: './files/airflow-packages-*' + - uses: actions/upload-artifact@v2 + name: Upload readmes + if: always() + with: + name: > + airflow-backport-readmes + path: './files/airflow-backport-readme*' trigger-tests: timeout-minutes: 5 @@ -177,7 +192,6 @@ jobs: run: ./scripts/ci/kubernetes/ci_run_kubernetes_tests.sh - uses: actions/upload-artifact@v2 name: Upload KinD logs - # Always run this, even if one of th previous steps failed. if: always() with: name: 'kind-logs-${{matrix.kube-mode}}-${{matrix.python-version}}-${{matrix.kubernetes-version}}' @@ -212,6 +226,12 @@ jobs: run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - name: "Tests" run: ./scripts/ci/testing/ci_run_airflow_testing.sh + - uses: actions/upload-artifact@v2 + name: Upload airflow logs + if: always() + with: + name: 'airflow-logs-${{matrix.test-type}}-${{matrix.python-version}}-${{matrix.postgres-version}}' + path: './files/airflow_logs*' tests-mysql: timeout-minutes: 80 @@ -242,6 +262,12 @@ jobs: run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - name: "Tests" run: ./scripts/ci/testing/ci_run_airflow_testing.sh + - uses: actions/upload-artifact@v2 + name: Upload airflow logs + if: always() + with: + name: 'airflow-logs-${{matrix.test-type}}-${{matrix.python-version}}-${{matrix.mysql-version}}' + path: './files/airflow_logs*' tests-sqlite: timeout-minutes: 80 @@ -256,8 +282,8 @@ jobs: env: BACKEND: sqlite PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} - TEST_TYPE: ${{ matrix.test-type }} RUN_TESTS: "true" + TEST_TYPE: ${{ matrix.test-type }} if: needs.trigger-tests.outputs.run-tests == 'true' || github.event_name != 'pull_request' steps: - uses: actions/checkout@v2 @@ -270,6 +296,12 @@ jobs: run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - name: "Tests" run: ./scripts/ci/testing/ci_run_airflow_testing.sh + - uses: actions/upload-artifact@v2 + name: Upload airflow logs + if: always() + with: + name: 'airflow-logs-${{matrix.test-type}}-${{matrix.python-version}}' + path: './files/airflow_logs*' helm-tests: timeout-minutes: 5 @@ -383,7 +415,7 @@ jobs: name: Upload constraint artifacts with: name: 'constraints-${{matrix.python-version}}' - path: 'files/constraints-${{matrix.python-version}}/constraints-${{matrix.python-version}}.txt' + path: './files/constraints-${{matrix.python-version}}/constraints-${{matrix.python-version}}.txt' constraints-push: timeout-minutes: 10 diff --git a/.github/workflows/quarantined.yaml b/.github/workflows/quarantined.yaml index 43852de106..7b3dc629b1 100644 --- a/.github/workflows/quarantined.yaml +++ b/.github/workflows/quarantined.yaml @@ -114,3 +114,9 @@ jobs: with: name: 'quarantined_tests' path: 'files/test_result.xml' + - uses: actions/upload-artifact@v2 + name: Upload airflow logs + if: always() + with: + name: airflow-logs-quarantined-${{matrix.python-version}}-${{matrix.kubernetes-version}} + path: './files/airflow_logs*' diff --git a/airflow/cli/commands/info_command.py b/airflow/cli/commands/info_command.py index 01f3f7d96e..487e05a696 100644 --- a/airflow/cli/commands/info_command.py +++ b/airflow/cli/commands/info_command.py @@ -391,7 +391,7 @@ class FileIoException(Exception): after=tenacity.after_log(log, logging.DEBUG), ) def _upload_text_to_fileio(content): - """Uload text file to File.io service and return lnk""" + """Upload text file to File.io service and return lnk""" resp = requests.post("https://file.io", files={"file": ("airflow-report.txt", content)}) if not resp.ok: raise FileIoException("Failed to send report to file.io service.") diff --git a/breeze b/breeze index 5807144a15..c122b17bb8 100755 --- a/breeze +++ b/breeze @@ -97,6 +97,9 @@ function setup_default_breeze_variables() { # By default we mount local Airflow sources MOUNT_LOCAL_SOURCES="true" + # By default we mount files folder + MOUNT_FILES="true" + # Holds last sub-command used - this is used by --help flag to print help for the command entered LAST_SUBCOMMAND="" @@ -492,6 +495,7 @@ function prepare_command_files() { MAIN_PROD_DOCKER_COMPOSE_FILE=${SCRIPTS_CI_DIR}/docker-compose/base.yml BACKEND_DOCKER_COMPOSE_FILE=${SCRIPTS_CI_DIR}/docker-compose/backend-${BACKEND}.yml LOCAL_DOCKER_COMPOSE_FILE=${SCRIPTS_CI_DIR}/docker-compose/local.yml + FILES_DOCKER_COMPOSE_FILE=${SCRIPTS_CI_DIR}/docker-compose/files.yml LOCAL_PROD_DOCKER_COMPOSE_FILE=${SCRIPTS_CI_DIR}/docker-compose/local-prod.yml REMOVE_SOURCES_DOCKER_COMPOSE_FILE=${SCRIPTS_CI_DIR}/docker-compose/remove-sources.yml FORWARD_CREDENTIALS_DOCKER_COMPOSE_FILE=${SCRIPTS_CI_DIR}/docker-compose/forward-credentials.yml @@ -504,6 +508,11 @@ function prepare_command_files() { COMPOSE_PROD_FILE=${COMPOSE_PROD_FILE}:${LOCAL_PROD_DOCKER_COMPOSE_FILE} fi + if [[ "${MOUNT_FILES}" != "false" ]]; then + COMPOSE_CI_FILE=${COMPOSE_CI_FILE}:${FILES_DOCKER_COMPOSE_FILE} + COMPOSE_PROD_FILE=${COMPOSE_PROD_FILE}:${FILES_DOCKER_COMPOSE_FILE} + fi + if [[ ${FORWARD_CREDENTIALS} == "true" ]]; then COMPOSE_CI_FILE=${COMPOSE_CI_FILE}:${FORWARD_CREDENTIALS_DOCKER_COMPOSE_FILE} COMPOSE_PROD_FILE=${COMPOSE_PROD_FILE}:${FORWARD_CREDENTIALS_DOCKER_COMPOSE_FILE} @@ -2125,8 +2134,10 @@ function run_breeze_command { enter_breeze) if [[ ${PRODUCTION_IMAGE} == "true" ]]; then "${BUILD_CACHE_DIR}/${LAST_DC_PROD_FILE}" run --service-ports --rm airflow "${@}" + "${SCRIPTS_CI_DIR}/tools/ci_fix_ownership.sh" else "${BUILD_CACHE_DIR}/${LAST_DC_CI_FILE}" run --service-ports --rm airflow "${@}" + "${SCRIPTS_CI_DIR}/tools/ci_fix_ownership.sh" "${SCRIPTS_CI_DIR}/tools/ci_clear_tmp.sh" fi ;; diff --git a/dev/BACKPORT_PACKAGES.md b/dev/BACKPORT_PACKAGES.md index 7ba77a6ea6..a5ed4d2fb3 100644 --- a/dev/BACKPORT_PACKAGES.md +++ b/dev/BACKPORT_PACKAGES.md @@ -131,7 +131,7 @@ The script generates two types of files: Note that our CI system builds the release notes for backport packages automatically with every build and current date - this way you might be sure the automated generation of the release notes continues to -work. You can also preview the generated readme files (by downloading artifacts uploaded to file.io). +work. You can also preview the generated readme files (by downloading artifacts from Github Actions). The script does not modify the README and CHANGES files if there is no change in the repo for that provider. # Preparing backport packages diff --git a/scripts/ci/backport_packages/ci_prepare_and_test_backport_packages.sh b/scripts/ci/backport_packages/ci_prepare_and_test_backport_packages.sh index 7982606de5..d756dac169 100755 --- a/scripts/ci/backport_packages/ci_prepare_and_test_backport_packages.sh +++ b/scripts/ci/backport_packages/ci_prepare_and_test_backport_packages.sh @@ -25,15 +25,9 @@ export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6} "${SCRIPTS_CI_DIR}/backport_packages/ci_test_backport_packages_install_separately.sh" "${SCRIPTS_CI_DIR}/backport_packages/ci_test_backport_packages_import_all_classes.sh" -cd "${AIRFLOW_SOURCES}/backport_packages" || exit 1 - DUMP_FILE="/tmp/airflow_provider_packages_$(date +"%Y%m%d-%H%M%S").tar.gz" cd "${AIRFLOW_SOURCES}/dist" || exit 1 tar -cvzf "${DUMP_FILE}" . echo "Packages are in dist and also tar-gzipped in ${DUMP_FILE}" - -if [[ "${CI:=false}" == "true" ]]; then - curl -F "file=@${DUMP_FILE}" https://file.io -fi diff --git a/scripts/ci/docker-compose/base.yml b/scripts/ci/docker-compose/base.yml index 9a364a22ae..7f54b206b7 100644 --- a/scripts/ci/docker-compose/base.yml +++ b/scripts/ci/docker-compose/base.yml @@ -49,6 +49,12 @@ services: - ENABLED_SYSTEMS - RUN_SYSTEM_TESTS - PYTHON_MAJOR_MINOR_VERSION + - HOST_USER_ID + - HOST_GROUP_ID + - HOST_HOME=${HOME} + - HOST_AIRFLOW_SOURCES=${AIRFLOW_SOURCES} + - HOST_OS + - PYTHONDONTWRITEBYTECODE volumes: # Pass docker to inside of the container so that Kind and Moto tests can use it. - /var/run/docker.sock:/var/run/docker.sock diff --git a/scripts/ci/docker-compose/ci.yml b/scripts/ci/docker-compose/ci.yml new file mode 100644 index 0000000000..1d0c6a4d6b --- /dev/null +++ b/scripts/ci/docker-compose/ci.yml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +version: "2.2" +services: + airflow: + environment: + - GITHUB_HEAD_REF + - GITHUB_REF + - GITHUB_ACTIONS + - GITHUB_SHA + - GITHUB_REPOSITORY + - GITHUB_RUN_ID + - GITHUB_TOKEN diff --git a/scripts/ci/docker-compose/files.yml b/scripts/ci/docker-compose/files.yml new file mode 100644 index 0000000000..5625ca6cfb --- /dev/null +++ b/scripts/ci/docker-compose/files.yml @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +version: "2.2" +services: + airflow: + volumes: + - ../../../files:/files:cached diff --git a/scripts/ci/docker-compose/local-prod.yml b/scripts/ci/docker-compose/local-prod.yml index 10db4e7d90..dce04f590e 100644 --- a/scripts/ci/docker-compose/local-prod.yml +++ b/scripts/ci/docker-compose/local-prod.yml @@ -29,7 +29,6 @@ services: - ../../../.github:/opt/airflow/.github:cached - ../../../.inputrc:/root/.inputrc:cached - ../../../.kube:/root/.kube:cached - - ../../../files:/files:cached - ../../../dist:/dist:cached - ../../../scripts/prod/entrypoint_prod.sh:/entrypoint:cached - ../../../setup.cfg:/opt/airflow/setup.cfg:cached diff --git a/scripts/ci/docker-compose/local.yml b/scripts/ci/docker-compose/local.yml index 3c692c861d..df49fb0385 100644 --- a/scripts/ci/docker-compose/local.yml +++ b/scripts/ci/docker-compose/local.yml @@ -45,7 +45,6 @@ services: - ../../../dags:/opt/airflow/dags:cached - ../../../dev:/opt/airflow/dev:cached - ../../../docs:/opt/airflow/docs:cached - - ../../../files:/files:cached - ../../../dist:/dist:cached - ../../../hooks:/opt/airflow/hooks:cached - ../../../logs:/root/airflow/logs:cached @@ -60,12 +59,5 @@ services: - ../../../tmp:/tmp:cached - ../../../metastore_browser:/opt/airflow/metastore_browser:cached # END automatically generated volumes from LOCAL_MOUNTS in _local_mounts.sh - environment: - - HOST_USER_ID - - HOST_GROUP_ID - - HOST_HOME=${HOME} - - HOST_AIRFLOW_SOURCES=${AIRFLOW_SOURCES} - - HOST_OS - - PYTHONDONTWRITEBYTECODE ports: - "${WEBSERVER_HOST_PORT}:8080" diff --git a/scripts/ci/in_container/_in_container_utils.sh b/scripts/ci/in_container/_in_container_utils.sh index 18af0b2e93..56d0fc9326 100644 --- a/scripts/ci/in_container/_in_container_utils.sh +++ b/scripts/ci/in_container/_in_container_utils.sh @@ -248,31 +248,18 @@ function setup_kerberos() { } function dump_airflow_logs() { + DUMP_FILE=/files/airflow_logs_$(date "+%Y-%m-%d")_${CI_BUILD_ID:="default"}_${CI_JOB_ID:="default"}.log.tar.gz echo "###########################################################################################" echo " Dumping logs from all the airflow tasks" echo "###########################################################################################" - pushd /root/airflow/ || exit 1 - tar -czf "${1}" logs + pushd "${AIRFLOW_HOME}" || exit 1 + tar -czf "${DUMP_FILE}" logs + echo " Logs dumped to ${DUMP_FILE}" popd || exit 1 echo "###########################################################################################" } -function send_airflow_logs_to_file_io() { - echo "##############################################################################" - echo - echo " DUMPING LOG FILES FROM AIRFLOW AND SENDING THEM TO file.io" - echo - echo "##############################################################################" - DUMP_FILE=/tmp/$(date "+%Y-%m-%d")_airflow_${CI_BUILD_ID:="default"}_${CI_JOB_ID:="default"}.log.tar.gz - dump_airflow_logs "${DUMP_FILE}" - echo - echo " Logs saved to ${DUMP_FILE}" - echo - echo "##############################################################################" - curl -F "file=@${DUMP_FILE}" https://file.io -} - function install_released_airflow_version() { pip uninstall -y apache-airflow || true find /root/airflow/ -type f -print0 | xargs -0 rm -f -- diff --git a/scripts/ci/in_container/run_ci_tests.sh b/scripts/ci/in_container/run_ci_tests.sh index 6d3648026d..704577ead5 100755 --- a/scripts/ci/in_container/run_ci_tests.sh +++ b/scripts/ci/in_container/run_ci_tests.sh @@ -59,7 +59,7 @@ else fi if [[ ${CI:=} == "true" ]]; then - send_airflow_logs_to_file_io + dump_airflow_logs fi exit "${RES}" diff --git a/scripts/ci/in_container/run_prepare_backport_packages.sh b/scripts/ci/in_container/run_prepare_backport_packages.sh index c74fb1637c..a491c61f4b 100755 --- a/scripts/ci/in_container/run_prepare_backport_packages.sh +++ b/scripts/ci/in_container/run_prepare_backport_packages.sh @@ -170,16 +170,9 @@ fi popd -AIRFLOW_PACKAGES_TGZ_FILE="/tmp/airflow-packages-$(date +"%Y%m%d-%H%M%S")-${VERSION_SUFFIX_FOR_SVN}${VERSION_SUFFIX_FOR_PYPI}.tar.gz" +AIRFLOW_PACKAGES_TGZ_FILE="/files/airflow-packages-$(date +"%Y%m%d-%H%M%S")-${VERSION_SUFFIX_FOR_SVN}${VERSION_SUFFIX_FOR_PYPI}.tar.gz" tar -cvzf "${AIRFLOW_PACKAGES_TGZ_FILE}" dist/*.whl dist/*.tar.gz echo echo "Airflow packages are in dist folder and tar-gzipped in ${AIRFLOW_PACKAGES_TGZ_FILE}" echo -if [[ "${CI:=false}" == "true" ]]; then - echo - echo "Sending all airflow packages to file.io" - echo - curl -F "file=@${AIRFLOW_PACKAGES_TGZ_FILE}" https://file.io - echo -fi diff --git a/scripts/ci/in_container/run_prepare_backport_readme.sh b/scripts/ci/in_container/run_prepare_backport_readme.sh index 99c818302f..7af356519e 100755 --- a/scripts/ci/in_container/run_prepare_backport_readme.sh +++ b/scripts/ci/in_container/run_prepare_backport_readme.sh @@ -41,7 +41,7 @@ cd "${AIRFLOW_SOURCES}/backport_packages" || exit 1 python3 setup_backport_packages.py update-package-release-notes "$@" -AIRFLOW_BACKPORT_README_TGZ_FILE="/dist/airflow-backport-readme-$(date +"%Y-%m-%d-%H.%M.%S").tar.gz" +AIRFLOW_BACKPORT_README_TGZ_FILE="/files/airflow-backport-readme-$(date +"%Y-%m-%d-%H.%M.%S").tar.gz" cd "${AIRFLOW_SOURCES}" || exit 1 @@ -50,10 +50,3 @@ find airflow/providers \( -name 'README.md' -o -name 'PROVIDERS_CHANGES*' \) -pr echo echo "Airflow readme for backport packages are tar-gzipped in ${AIRFLOW_BACKPORT_README_TGZ_FILE}" echo -if [[ "${CI:=false}" == "true" ]]; then - echo - echo "Sending all airflow packages to file.io" - echo - curl -F "file=@${AIRFLOW_PACKAGES_TGZ_FILE}" https://file.io - echo -fi diff --git a/scripts/ci/in_container/run_system_tests.sh b/scripts/ci/in_container/run_system_tests.sh index 8cb3c3e440..e3d7c7fd9a 100755 --- a/scripts/ci/in_container/run_system_tests.sh +++ b/scripts/ci/in_container/run_system_tests.sh @@ -47,7 +47,7 @@ if [[ "${RES}" == "0" && ${CI} == "true" ]]; then fi if [[ ${CI} == "true" ]]; then - send_airflow_logs_to_file_io + dump_airflow_logs fi in_container_script_end diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh index 08a6b694c8..3cbdbe6620 100644 --- a/scripts/ci/libraries/_initialization.sh +++ b/scripts/ci/libraries/_initialization.sh @@ -138,6 +138,7 @@ function initialize_common_environment { print_info read -r -a EXTRA_DOCKER_FLAGS <<< "$(convert_local_mounts_to_docker_params)" + EXTRA_DOCKER_FLAGS+=("-v" "${AIRFLOW_SOURCES}/files:/files" ) else print_info print_info "Skip mounting host volumes to Docker" diff --git a/scripts/ci/libraries/_local_mounts.sh b/scripts/ci/libraries/_local_mounts.sh index 78634d8946..23c7bb72e8 100644 --- a/scripts/ci/libraries/_local_mounts.sh +++ b/scripts/ci/libraries/_local_mounts.sh @@ -41,7 +41,6 @@ function generate_local_mounts_list { "$prefix"dags:/opt/airflow/dags:cached "$prefix"dev:/opt/airflow/dev:cached "$prefix"docs:/opt/airflow/docs:cached - "$prefix"files:/files:cached "$prefix"dist:/dist:cached "$prefix"hooks:/opt/airflow/hooks:cached "$prefix"logs:/root/airflow/logs:cached diff --git a/scripts/ci/testing/ci_run_airflow_testing.sh b/scripts/ci/testing/ci_run_airflow_testing.sh index 091d87f3fc..8d37b6d914 100755 --- a/scripts/ci/testing/ci_run_airflow_testing.sh +++ b/scripts/ci/testing/ci_run_airflow_testing.sh @@ -64,10 +64,13 @@ export BACKEND=${BACKEND:="sqlite"} # Whether necessary for airflow run local sources are mounted to docker export MOUNT_LOCAL_SOURCES=${MOUNT_LOCAL_SOURCES:="false"} -# whethere verbose output should be produced +# Whether files folder is mounted to docker +export MOUNT_FILES=${MOUNT_FILES:="true"} + +# whether verbose output should be produced export VERBOSE=${VERBOSE:="false"} -# whethere verbose commadns output (set-x) should be used +# whether verbose commands output (set -x) should be used export VERBOSE_COMMANDS=${VERBOSE_COMMANDS:="false"} # Forwards host credentials to the container @@ -76,10 +79,18 @@ export FORWARD_CREDENTIALS=${FORWARD_CREDENTIALS:="false"} # Installs different airflow version than current from the sources export INSTALL_AIRFLOW_VERSION=${INSTALL_AIRFLOW_VERSION:=""} +DOCKER_COMPOSE_LOCAL=() + if [[ ${MOUNT_LOCAL_SOURCES} == "true" ]]; then - DOCKER_COMPOSE_LOCAL=("-f" "${SCRIPTS_CI_DIR}/docker-compose/local.yml") -else - DOCKER_COMPOSE_LOCAL=() + DOCKER_COMPOSE_LOCAL+=("-f" "${SCRIPTS_CI_DIR}/docker-compose/local.yml") +fi + +if [[ ${MOUNT_FILES} == "true" ]]; then + DOCKER_COMPOSE_LOCAL+=("-f" "${SCRIPTS_CI_DIR}/docker-compose/files.yml") +fi + +if [[ ${CI} == "true" ]]; then + DOCKER_COMPOSE_LOCAL+=("-f" "${SCRIPTS_CI_DIR}/docker-compose/ci.yml") fi if [[ ${FORWARD_CREDENTIALS} == "true" ]]; then @@ -116,7 +127,6 @@ done RUN_INTEGRATION_TESTS=${RUN_INTEGRATION_TESTS:=""} - run_airflow_testing_in_docker "${@}" if [[ ${TEST_TYPE:=} == "Quarantined" ]]; then diff --git a/scripts/ci/tools/ci_clear_tmp.sh b/scripts/ci/tools/ci_clear_tmp.sh index 9a8cb4a6fc..ff5f601869 100755 --- a/scripts/ci/tools/ci_clear_tmp.sh +++ b/scripts/ci/tools/ci_clear_tmp.sh @@ -40,5 +40,6 @@ export HOST_OS docker-compose \ -f "${SCRIPTS_CI_DIR}/docker-compose/base.yml" \ -f "${SCRIPTS_CI_DIR}/docker-compose/local.yml" \ + -f "${SCRIPTS_CI_DIR}/docker-compose/files.yml" \ run --entrypoint /bin/bash \ airflow -c /opt/airflow/scripts/ci/in_container/run_clear_tmp.sh diff --git a/scripts/ci/tools/ci_fix_ownership.sh b/scripts/ci/tools/ci_fix_ownership.sh index d3ae4bad1d..762f41e01b 100755 --- a/scripts/ci/tools/ci_fix_ownership.sh +++ b/scripts/ci/tools/ci_fix_ownership.sh @@ -41,6 +41,7 @@ export BACKEND="sqlite" docker-compose \ -f "${SCRIPTS_CI_DIR}/docker-compose/base.yml" \ -f "${SCRIPTS_CI_DIR}/docker-compose/local.yml" \ + -f "${SCRIPTS_CI_DIR}/docker-compose/files.yml" \ -f "${SCRIPTS_CI_DIR}/docker-compose/forward-credentials.yml" \ run --entrypoint /bin/bash \ airflow -c /opt/airflow/scripts/ci/in_container/run_fix_ownership.sh