github
/
incubator-airflow
зеркало из https://github.com/github/incubator-airflow.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

[AIRFLOW-6816] Simplified common functions in breeze scripts (#7430)

This commit is contained in:
Jarek Potiuk 2020-02-16 20:05:50 +01:00 коммит произвёл GitHub
Родитель 345a96894a
Коммит ff7034f39a
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
51 изменённых файлов: 515 добавлений и 759 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

4
.pre-commit-config.yaml
Просмотреть файл

@ -285,14 +285,14 @@ repos:
language: system
entry: "./scripts/ci/pre_commit_mypy.sh"
files: \.py$
exclude: ^airflow/_vendor/.*$
exclude: ^airflow/_vendor/.*$|^dev
pass_filenames: false
- id: pylint
name: Run pylint for main sources
language: system
entry: "./scripts/ci/pre_commit_pylint_main.sh"
files: \.py$
exclude: ^tests/.*\.py$|^airflow/_vendor/.*|^scripts/.*\.py$
exclude: ^tests/.*\.py$|^airflow/_vendor/.*|^scripts/.*\.py$|^dev
pass_filenames: true
require_serial: true # Pylint tests should be run in one chunk to detect all cycles
- id: pylint-tests

3
.rat-excludes
Просмотреть файл

@ -33,6 +33,9 @@ CHANGELOG.txt
unittests.cfg
logs
.bash_aliases
venv
files
airflow.iml
# Generated doc files
.*html

34
breeze
Просмотреть файл

@ -37,15 +37,28 @@ mkdir -pv "${BUILD_CACHE_DIR}"
mkdir -pv "${TMP_DIR}"
mkdir -pv "${FILES_DIR}"
# Note - we do not use __script_init.sh here because it can only be used from within
# the CI directory and we need to overrride PYTHON_VERSION based on what we store
# in the .build directory
# Beginning of the initialisation here
# shellcheck source=scripts/ci/_utils.sh
. "${SCRIPTS_CI_DIR}/_utils.sh"
initialize_breeze_environment
export PYTHON_VERSION="${PYTHON_VERSION:=$(read_from_file PYTHON_VERSION)}"
basic_sanity_checks
script_start
trap script_end EXIT
# End of initialisation here
# Sets width of the screen
SEPARATOR_WIDTH="$(tput cols)"
@ -113,9 +126,6 @@ export AIRFLOW_CONTAINER_USE_PULLED_IMAGES_CACHE=${AIRFLOW_CONTAINER_USE_PULLED_
# By default we do not push images. This can be overridden by -u flag.
export AIRFLOW_CONTAINER_PUSH_IMAGES=${AIRFLOW_CONTAINER_PUSH_IMAGES:="false"}
# For local builds we fix file permissions only for setup-related files
export AIRFLOW_FIX_PERMISSIONS=${AIRFLOW_FIX_PERMISSIONS:="setup"}
# Skip building full CI image locally
export AIRFLOW_CONTAINER_SKIP_CI_IMAGE="false"
@ -630,6 +640,9 @@ do
shift ;;
-b|--build-only)
ENTER_ENVIRONMENT="false"
# if you want to build an image - assume you want to build it :)
export FORCE_ANSWER_TO_QUESTIONS="yes"
# and assume you want to build it no matter if it is needed
export AIRFLOW_CONTAINER_FORCE_DOCKER_BUILD="true"
echo "Only build. Do not enter airflow-testing container"
echo
@ -671,6 +684,8 @@ do
echo "Force build images"
echo
export AIRFLOW_CONTAINER_FORCE_DOCKER_BUILD="true"
# if you want to force build an image - assume you want to build it :)
export FORCE_ANSWER_TO_QUESTIONS="yes"
shift ;;
-R|--force-clean-images)
echo "Clean build of images without cache"
@ -692,17 +707,6 @@ do
export AIRFLOW_CONTAINER_FORCE_DOCKER_BUILD="true"
shift ;;
-u|--push-images)
if [[ "${AIRFLOW_FIX_PERMISSIONS}" != "all" ]]; then
echo >&2
echo >&2 "ERROR: Disable fix permissions when pushing"
echo >&2
echo >&2 "You cannot push images if you have AIRFLOW_FIX_PERMISSIONS set to other value than 'all'"
echo >&2 "Your docker context is most likely wrong in this case"
echo >&2 "You need to set AIRFLOW_FIX_PERMISSIONS to false"
echo >&2 "And run the build again"
echo >&2
exit 1
fi
echo
echo "Pushing images to DockerHub"
echo
@ -1198,5 +1202,3 @@ else
run_docs
fi
fi
script_end

4
hooks/build
Просмотреть файл

@ -200,10 +200,6 @@ export AIRFLOW_CONTAINER_SKIP_LATEST_PYTHON_PULL=${AIRFLOW_CONTAINER_SKIP_LATEST
export AIRFLOW_CONTAINER_SKIP_CI_IMAGE=${AIRFLOW_CONTAINER_SKIP_CI_IMAGE:="false"}
echo "Skip CI image: ${AIRFLOW_CONTAINER_SKIP_CI_IMAGE}"
# Fixes permissions for git-checked out files. This is needed to have consistent build cache across
# Dockerhub, TravisCI and locally checked out code
export AIRFLOW_FIX_PERMISSIONS=${AIRFLOW_FIX_PERMISSIONS:="all"}
echo "Fixing permissions: ${AIRFLOW_FIX_PERMISSIONS}"
export AIRFLOW_CONTAINER_CI_OPTIMISED_BUILD=${AIRFLOW_CONTAINER_CI_OPTIMISED_BUILD:="true"}
echo "The build optimised for CI: ${AIRFLOW_CONTAINER_CI_OPTIMISED_BUILD}"

34
scripts/ci/_script_init.sh Executable file
Просмотреть файл

@ -0,0 +1,34 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -euo pipefail
# This should only be sourced from CI directory!
# shellcheck source=scripts/ci/_utils.sh
. "$( dirname "${BASH_SOURCE[0]}" )/_utils.sh"
MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
export MY_DIR
initialize_breeze_environment
basic_sanity_checks
script_start
trap script_end EXIT

391
scripts/ci/_utils.sh
Просмотреть файл

@ -19,60 +19,109 @@
# Assume all the scripts are sourcing the _utils.sh from the scripts/ci directory
# and MY_DIR variable is set to this directory. It can be overridden however
if [[ ${VERBOSE:=} == "true" ]]; then
set -x
else
set +x
fi
declare -a AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS
AIRFLOW_SOURCES=${AIRFLOW_SOURCES:=$(cd "${MY_DIR}/../../" && pwd)}
export AIRFLOW_SOURCES
function check_verbose_setup {
if [[ ${VERBOSE:=} == "true" ]]; then
set -x
else
set +x
fi
}
BUILD_CACHE_DIR="${AIRFLOW_SOURCES}/.build"
export BUILD_CACHE_DIR
LAST_FORCE_ANSWER_FILE="${BUILD_CACHE_DIR}/last_force_answer.sh"
function initialize_breeze_environment {
AIRFLOW_SOURCES=${AIRFLOW_SOURCES:=$(cd "${MY_DIR}/../../" && pwd)}
export AIRFLOW_SOURCES
IMAGES_TO_CHECK=("CI")
export IMAGES_TO_CHECK
BUILD_CACHE_DIR="${AIRFLOW_SOURCES}/.build"
export BUILD_CACHE_DIR
mkdir -p "${AIRFLOW_SOURCES}/.mypy_cache"
mkdir -p "${AIRFLOW_SOURCES}/logs"
mkdir -p "${AIRFLOW_SOURCES}/tmp"
LAST_FORCE_ANSWER_FILE="${BUILD_CACHE_DIR}/last_force_answer.sh"
# shellcheck source=common/_autodetect_variables.sh
. "${AIRFLOW_SOURCES}/common/_autodetect_variables.sh"
# shellcheck source=common/_files_for_rebuild_check.sh
. "${AIRFLOW_SOURCES}/common/_files_for_rebuild_check.sh"
IMAGES_TO_CHECK=("CI")
export IMAGES_TO_CHECK
# Default branch name for triggered builds is the one configured in default branch
export AIRFLOW_CONTAINER_BRANCH_NAME=${AIRFLOW_CONTAINER_BRANCH_NAME:=${DEFAULT_BRANCH}}
# Create directories if needed
mkdir -p "${AIRFLOW_SOURCES}/.mypy_cache"
mkdir -p "${AIRFLOW_SOURCES}/logs"
mkdir -p "${AIRFLOW_SOURCES}/tmp"
mkdir -p "${AIRFLOW_SOURCES}/files"
mkdir -p "${AIRFLOW_SOURCES}/dist"
# Default port numbers for forwarded ports
export WEBSERVER_HOST_PORT=${WEBSERVER_HOST_PORT:="28080"}
export POSTGRES_HOST_PORT=${POSTGRES_HOST_PORT:="25433"}
export MYSQL_HOST_PORT=${MYSQL_HOST_PORT:="23306"}
# shellcheck source=common/_autodetect_variables.sh
. "${AIRFLOW_SOURCES}/common/_autodetect_variables.sh"
# shellcheck source=common/_files_for_rebuild_check.sh
. "${AIRFLOW_SOURCES}/common/_files_for_rebuild_check.sh"
# Do not push images from here by default (push them directly from the build script on Dockerhub)
export AIRFLOW_CONTAINER_PUSH_IMAGES=${AIRFLOW_CONTAINER_PUSH_IMAGES:="false"}
# Default branch name for triggered builds is the one configured in default branch
export AIRFLOW_CONTAINER_BRANCH_NAME=${AIRFLOW_CONTAINER_BRANCH_NAME:=${DEFAULT_BRANCH}}
# Disable writing .pyc files - slightly slower imports but not messing around when switching
# Python version and avoids problems with root-owned .pyc files in host
export PYTHONDONTWRITEBYTECODE=${PYTHONDONTWRITEBYTECODE:="true"}
# Default port numbers for forwarded ports
export WEBSERVER_HOST_PORT=${WEBSERVER_HOST_PORT:="28080"}
export POSTGRES_HOST_PORT=${POSTGRES_HOST_PORT:="25433"}
export MYSQL_HOST_PORT=${MYSQL_HOST_PORT:="23306"}
# By default we assume the kubernetes cluster is not being started
export ENABLE_KIND_CLUSTER=${ENABLE_KIND_CLUSTER:="false"}
#
# Sets mounting of host volumes to container for static checks
# unless AIRFLOW_MOUNT_HOST_VOLUMES_FOR_STATIC_CHECKS is not true
#
# Note that this cannot be function because we need the AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS array variable
#
AIRFLOW_MOUNT_HOST_VOLUMES_FOR_STATIC_CHECKS=${AIRFLOW_MOUNT_HOST_VOLUMES_FOR_STATIC_CHECKS:="true"}
# Do not push images from here by default (push them directly from the build script on Dockerhub)
export AIRFLOW_CONTAINER_PUSH_IMAGES=${AIRFLOW_CONTAINER_PUSH_IMAGES:="false"}
# If this variable is set, we mount the whole sources directory to the host rather than
# selected volumes
AIRFLOW_MOUNT_SOURCE_DIR_FOR_STATIC_CHECKS=${AIRFLOW_MOUNT_SOURCE_DIR_FOR_STATIC_CHECKS="false"}
# Disable writing .pyc files - slightly slower imports but not messing around when switching
# Python version and avoids problems with root-owned .pyc files in host
export PYTHONDONTWRITEBYTECODE=${PYTHONDONTWRITEBYTECODE:="true"}
# By default we assume the kubernetes cluster is not being started
export ENABLE_KIND_CLUSTER=${ENABLE_KIND_CLUSTER:="false"}
#
# Sets mounting of host volumes to container for static checks
# unless AIRFLOW_MOUNT_HOST_VOLUMES_FOR_STATIC_CHECKS is not true
#
# Note that this cannot be function because we need the AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS array variable
#
AIRFLOW_MOUNT_HOST_VOLUMES_FOR_STATIC_CHECKS=${AIRFLOW_MOUNT_HOST_VOLUMES_FOR_STATIC_CHECKS:="true"}
export AIRFLOW_MOUNT_HOST_VOLUMES_FOR_STATIC_CHECKS
# If this variable is set, we mount the whole sources directory to the host rather than
# selected volumes
AIRFLOW_MOUNT_SOURCE_DIR_FOR_STATIC_CHECKS=${AIRFLOW_MOUNT_SOURCE_DIR_FOR_STATIC_CHECKS="false"}
export AIRFLOW_MOUNT_SOURCE_DIR_FOR_STATIC_CHECKS
# Set host user id to current user
HOST_USER_ID="$(id -ur)"
export HOST_USER_ID
# Set host group id to current group
HOST_GROUP_ID="$(id -gr)"
export HOST_GROUP_ID
if [[ ${AIRFLOW_MOUNT_SOURCE_DIR_FOR_STATIC_CHECKS} == "true" ]]; then
print_info
print_info "Mount whole airflow source directory for static checks (make sure all files are in container)"
print_info
AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS=( \
"-v" "${AIRFLOW_SOURCES}:/opt/airflow" \
"--env" "PYTHONDONTWRITEBYTECODE" \
)
elif [[ ${AIRFLOW_MOUNT_HOST_VOLUMES_FOR_STATIC_CHECKS} == "true" ]]; then
print_info
print_info "Mounting necessary host volumes to Docker"
print_info
AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS=()
while IFS= read -r LINE; do
AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS+=( "${LINE}")
done < <(convert_docker_mounts_to_docker_params)
else
print_info
print_info "Skip mounting host volumes to Docker"
print_info
AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS=( \
"--env" "PYTHONDONTWRITEBYTECODE" \
)
fi
export AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS
}
function print_info() {
if [[ ${AIRFLOW_CI_SILENT:="false"} != "true" || ${VERBOSE:="false"} == "true" ]]; then
@ -146,36 +195,6 @@ function sanitize_mounted_files() {
sanitize_file "${AIRFLOW_SOURCES}/.inputrc"
}
declare -a AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS
if [[ ${AIRFLOW_MOUNT_SOURCE_DIR_FOR_STATIC_CHECKS} == "true" ]]; then
print_info
print_info "Mount whole airflow source directory for static checks (make sure all files are in container)"
print_info
AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS=( \
"-v" "${AIRFLOW_SOURCES}:/opt/airflow" \
"--env" "PYTHONDONTWRITEBYTECODE" \
)
elif [[ ${AIRFLOW_MOUNT_HOST_VOLUMES_FOR_STATIC_CHECKS} == "true" ]]; then
print_info
print_info "Mounting necessary host volumes to Docker"
print_info
AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS=()
while IFS= read -r LINE; do
AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS+=( "${LINE}")
done < <(convert_docker_mounts_to_docker_params)
else
print_info
print_info "Skip mounting host volumes to Docker"
print_info
AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS=( \
"--env" "PYTHONDONTWRITEBYTECODE" \
)
fi
export AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS
#
# Creates cache directory where we will keep temporary files needed for the build
#
@ -558,18 +577,6 @@ function rebuild_ci_image_if_needed() {
}
#
# Cleans up the CI image
#
function cleanup_ci_image() {
export AIRFLOW_CONTAINER_SKIP_CI_IMAGE="false"
export AIRFLOW_CONTAINER_CLEANUP_IMAGES="true"
export THE_IMAGE_TYPE="CI"
rebuild_image_if_needed
}
#
# Starts the script/ If VERBOSE variable is set to true, it enables verbose output of commands executed
# Also prints some useful diagnostics information at start of the script
@ -602,6 +609,13 @@ function script_start {
# Disables verbosity in the script
#
function script_end {
#shellcheck disable=2181
EXIT_CODE=$?
if [[ ${EXIT_CODE} != 0 ]]; then
echo "###########################################################################################"
echo " EXITING WITH STATUS CODE ${EXIT_CODE}"
echo "###########################################################################################"
fi
if [[ ${VERBOSE:=} == "true" ]]; then
set +x
fi
@ -609,7 +623,8 @@ function script_end {
RUN_SCRIPT_TIME=$((END_SCRIPT_TIME-START_SCRIPT_TIME))
print_info
print_info "Finished the script $(basename "$0")"
print_info "It took ${RUN_SCRIPT_TIME} seconds"
print_info "Elapsed time spent in the script: ${RUN_SCRIPT_TIME} seconds"
print_info "Exit code ${EXIT_CODE}"
print_info
remove_cache_directory
}
@ -634,185 +649,6 @@ function basic_sanity_checks() {
sanitize_mounted_files
}
function run_flake8() {
FILES=("$@")
if [[ "${#FILES[@]}" == "0" ]]; then
docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \
--entrypoint "/usr/local/bin/dumb-init" \
--env PYTHONDONTWRITEBYTECODE \
--env AIRFLOW_CI_VERBOSE="${VERBOSE}" \
--env AIRFLOW_CI_SILENT \
--env HOST_USER_ID="$(id -ur)" \
--env HOST_GROUP_ID="$(id -gr)" \
--rm \
"${AIRFLOW_CI_IMAGE}" \
"--" "/opt/airflow/scripts/ci/in_container/run_flake8.sh" \
| tee -a "${OUTPUT_LOG}"
else
docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \
--entrypoint "/usr/local/bin/dumb-init" \
--env PYTHONDONTWRITEBYTECODE \
--env AIRFLOW_CI_VERBOSE="${VERBOSE}" \
--env AIRFLOW_CI_SILENT \
--env HOST_USER_ID="$(id -ur)" \
--env HOST_GROUP_ID="$(id -gr)" \
--rm \
"${AIRFLOW_CI_IMAGE}" \
"--" "/opt/airflow/scripts/ci/in_container/run_flake8.sh" "${FILES[@]}" \
| tee -a "${OUTPUT_LOG}"
fi
}
function run_bats_tests() {
FILES=("$@")
if [[ "${#FILES[@]}" == "0" ]]; then
docker run --workdir /airflow -v "$(pwd):/airflow" --rm \
bats/bats:latest --tap -r /airflow/tests/bats | tee -a "${OUTPUT_LOG}"
else
docker run --workdir /airflow -v "$(pwd):/airflow" --rm \
bats/bats:latest --tap -r "${FILES[@]}" | tee -a "${OUTPUT_LOG}"
fi
}
function run_docs() {
docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" -t \
--entrypoint "/usr/local/bin/dumb-init" \
--env PYTHONDONTWRITEBYTECODE \
--env AIRFLOW_CI_VERBOSE="${VERBOSE}" \
--env AIRFLOW_CI_SILENT \
--env HOST_USER_ID="$(id -ur)" \
--env HOST_GROUP_ID="$(id -gr)" \
--rm \
"${AIRFLOW_CI_IMAGE}" \
"--" "/opt/airflow/docs/build.sh" \
| tee -a "${OUTPUT_LOG}"
}
function run_check_license() {
docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" -t \
--entrypoint "/usr/local/bin/dumb-init" \
--env PYTHONDONTWRITEBYTECODE \
--env AIRFLOW_CI_VERBOSE="${VERBOSE}" \
--env AIRFLOW_CI_SILENT \
--env HOST_USER_ID="$(id -ur)" \
--env HOST_GROUP_ID="$(id -gr)" \
--rm \
"${AIRFLOW_CI_IMAGE}" \
"--" "/opt/airflow/scripts/ci/in_container/run_check_licence.sh" \
| tee -a "${OUTPUT_LOG}"
}
function run_mypy() {
FILES=("$@")
if [[ "${#FILES[@]}" == "0" ]]; then
FILES=(airflow tests docs)
fi
docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \
--entrypoint "/usr/local/bin/dumb-init" \
--env PYTHONDONTWRITEBYTECODE \
--env AIRFLOW_CI_VERBOSE="${VERBOSE}" \
--env AIRFLOW_CI_SILENT \
--env HOST_USER_ID="$(id -ur)" \
--env HOST_GROUP_ID="$(id -gr)" \
"-v" "${AIRFLOW_SOURCES}/.mypy_cache:/opt/airflow/.mypy_cache" \
--rm \
"${AIRFLOW_CI_IMAGE}" \
"--" "/opt/airflow/scripts/ci/in_container/run_mypy.sh" "${FILES[@]}" \
| tee -a "${OUTPUT_LOG}"
}
function run_pylint_main() {
FILES=("$@")
if [[ "${#FILES[@]}" == "0" ]]; then
docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \
--entrypoint "/usr/local/bin/dumb-init" \
--env PYTHONDONTWRITEBYTECODE \
--env AIRFLOW_CI_VERBOSE="${VERBOSE}" \
--env AIRFLOW_CI_SILENT \
--env HOST_USER_ID="$(id -ur)" \
--env HOST_GROUP_ID="$(id -gr)" \
--rm \
"${AIRFLOW_CI_IMAGE}" \
"--" "/opt/airflow/scripts/ci/in_container/run_pylint_main.sh" \
| tee -a "${OUTPUT_LOG}"
else
docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \
--entrypoint "/usr/local/bin/dumb-init" \
--env PYTHONDONTWRITEBYTECODE \
--env AIRFLOW_CI_VERBOSE="${VERBOSE}" \
--env AIRFLOW_CI_SILENT \
--env HOST_USER_ID="$(id -ur)" \
--env HOST_GROUP_ID="$(id -gr)" \
--rm \
"${AIRFLOW_CI_IMAGE}" \
"--" "/opt/airflow/scripts/ci/in_container/run_pylint_main.sh" "${FILES[@]}" \
| tee -a "${OUTPUT_LOG}"
fi
}
function run_pylint_tests() {
FILES=("$@")
if [[ "${#FILES[@]}" == "0" ]]; then
docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \
--entrypoint "/usr/local/bin/dumb-init" \
--env PYTHONDONTWRITEBYTECODE \
--env AIRFLOW_CI_VERBOSE="${VERBOSE}" \
--env AIRFLOW_CI_SILENT \
--env HOST_USER_ID="$(id -ur)" \
--env HOST_GROUP_ID="$(id -gr)" \
--rm \
"${AIRFLOW_CI_IMAGE}" \
"--" "/opt/airflow/scripts/ci/in_container/run_pylint_tests.sh" \
| tee -a "${OUTPUT_LOG}"
else
docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \
--entrypoint "/usr/local/bin/dumb-init" \
--env PYTHONDONTWRITEBYTECODE \
--env AIRFLOW_CI_VERBOSE="${VERBOSE}" \
--env AIRFLOW_CI_SILENT \
--env HOST_USER_ID="$(id -ur)" \
--env HOST_GROUP_ID="$(id -gr)" \
--rm \
"${AIRFLOW_CI_IMAGE}" \
"--" "/opt/airflow/scripts/ci/in_container/run_pylint_tests.sh" "${FILES[@]}" \
| tee -a "${OUTPUT_LOG}"
fi
}
function run_docker_lint() {
FILES=("$@")
if [[ "${#FILES[@]}" == "0" ]]; then
echo
echo "Running docker lint for all Dockerfiles"
echo
docker run \
-v "$(pwd):/root" \
-w /root \
--rm \
hadolint/hadolint /bin/hadolint Dockerfile*
echo
echo "Docker pylint completed with no errors"
echo
else
echo
echo "Running docker lint for $*"
echo
docker run \
-v "$(pwd):/root" \
-w /root \
--rm \
hadolint/hadolint /bin/hadolint "$@"
echo
echo "Docker pylint completed with no errors"
echo
fi
}
function filter_out_files_from_pylint_todo_list() {
FILTERED_FILES=()
set +e
@ -830,19 +666,6 @@ function filter_out_files_from_pylint_todo_list() {
export FILTERED_FILES
}
function refresh_pylint_todo() {
docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \
--env PYTHONDONTWRITEBYTECODE \
--env AIRFLOW_CI_VERBOSE="${VERBOSE}" \
--env AIRFLOW_CI_SILENT \
--env HOST_USER_ID="$(id -ur)" \
--env HOST_GROUP_ID="$(id -gr)" \
--rm \
"${AIRFLOW_CI_IMAGE}" \
/opt/airflow/scripts/ci/in_container/refresh_pylint_todo.sh \
| tee -a "${OUTPUT_LOG}"
}
function rebuild_all_images_if_needed_and_confirmed() {
AIRFLOW_CONTAINER_DOCKER_BUILD_NEEDED="false"
IMAGES_TO_REBUILD=()
@ -1021,3 +844,17 @@ function check_for_allowed_params {
fi
save_to_file "${_VARIABLE_NAME}"
}
function run_docs() {
docker run "${EXTRA_DOCKER_FLAGS[@]}" -t \
--entrypoint "/usr/local/bin/dumb-init" \
--env PYTHONDONTWRITEBYTECODE \
--env AIRFLOW_CI_VERBOSE="${VERBOSE}" \
--env AIRFLOW_CI_SILENT \
--env HOST_USER_ID="$(id -ur)" \
--env HOST_GROUP_ID="$(id -gr)" \
--rm \
"${AIRFLOW_CI_IMAGE}" \
"--" "/opt/airflow/docs/build.sh" \
| tee -a "${OUTPUT_LOG}"
}

26
scripts/ci/ci_bat_tests.sh
Просмотреть файл

@ -16,21 +16,15 @@
# specific language governing permissions and limitations
# under the License.
set -euo pipefail
MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
export AIRFLOW_CI_SILENT=${AIRFLOW_CI_SILENT:="true"}
export PYTHON_VERSION=${PYTHON_VERSION:-3.6}
# shellcheck source=scripts/ci/_utils.sh
. "${MY_DIR}/_utils.sh"
basic_sanity_checks
script_start
function run_bats_tests() {
FILES=("$@")
if [[ "${#FILES[@]}" == "0" ]]; then
docker run --workdir /airflow -v "$(pwd):/airflow" --rm \
bats/bats:latest --tap -r /airflow/tests/bats
else
docker run --workdir /airflow -v "$(pwd):/airflow" --rm \
bats/bats:latest --tap -r "${FILES[@]}"
fi
}
run_bats_tests "$@"
script_end

19
scripts/ci/ci_before_install.sh
Просмотреть файл

@ -15,25 +15,12 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -x
set -xeuo pipefail
MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
# shellcheck source=scripts/ci/_utils.sh
. "${MY_DIR}/_utils.sh"
basic_sanity_checks
script_start
# shellcheck source=scripts/ci/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
build_image_on_ci
KUBERNETES_MODE=${KUBERNETES_MODE:=""}
mkdir -p "${AIRFLOW_SOURCES}/files"
# We need newer version of six for Travis as they bundle 1.11.0 version
sudo pip install pre-commit 'six~=1.14'
script_end

29
scripts/ci/ci_check_license.sh
Просмотреть файл

@ -15,24 +15,27 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -uo pipefail
MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
export AIRFLOW_CI_SILENT=${AIRFLOW_CI_SILENT:="true"}
export AIRFLOW_MOUNT_SOURCE_DIR_FOR_STATIC_CHECKS="true"
export AIRFLOW_CI_SILENT=${AIRFLOW_CI_SILENT:="true"}
export PYTHON_VERSION=${PYTHON_VERSION:-3.6}
# shellcheck source=scripts/ci/_utils.sh
. "${MY_DIR}/_utils.sh"
# shellcheck source=scripts/ci/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
basic_sanity_checks
script_start
function run_check_license() {
docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" -t \
--entrypoint "/usr/local/bin/dumb-init" \
--env PYTHONDONTWRITEBYTECODE \
--env AIRFLOW_CI_VERBOSE="${VERBOSE}" \
--env AIRFLOW_CI_SILENT \
--env HOST_USER_ID="$(id -ur)" \
--env HOST_GROUP_ID="$(id -gr)" \
--rm \
"${AIRFLOW_CI_IMAGE}" \
"--" "/opt/airflow/scripts/ci/in_container/run_check_licence.sh" \
| tee -a "${OUTPUT_LOG}"
}
rebuild_ci_image_if_needed
run_check_license
script_end

20
scripts/ci/ci_cleanup.sh
Просмотреть файл

@ -15,21 +15,19 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# shellcheck source=scripts/ci/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
#
# Pulls and rebuilds the full CI image used for testing
# Cleans up the CI image
#
set -euo pipefail
function cleanup_ci_image() {
export AIRFLOW_CONTAINER_SKIP_CI_IMAGE="false"
export AIRFLOW_CONTAINER_CLEANUP_IMAGES="true"
MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
export THE_IMAGE_TYPE="CI"
# shellcheck source=scripts/ci/_utils.sh
. "${MY_DIR}/_utils.sh"
basic_sanity_checks
script_start
rebuild_image_if_needed
}
cleanup_ci_image
script_end

16
scripts/ci/ci_docs.sh
Просмотреть файл

@ -15,24 +15,12 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -euo pipefail
MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
export AIRFLOW_CI_SILENT=${AIRFLOW_CI_SILENT:="true"}
export PYTHON_VERSION=${PYTHON_VERSION:-3.6}
# shellcheck source=scripts/ci/_utils.sh
. "${MY_DIR}/_utils.sh"
basic_sanity_checks
script_start
# shellcheck source=scripts/ci/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
rebuild_ci_image_if_needed
run_docs
script_end

21
scripts/ci/ci_fix_ownership.sh
Просмотреть файл

@ -20,30 +20,15 @@
# Fixes ownership for files created inside container (files owned by root will be owned by host user)
#
set -euo pipefail
MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
# shellcheck source=scripts/ci/_utils.sh
. "${MY_DIR}/_utils.sh"
basic_sanity_checks
script_start
export PYTHON_VERSION=${PYTHON_VERSION:-3.6}
# shellcheck source=scripts/ci/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
export AIRFLOW_CI_IMAGE=\
${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_CONTAINER_BRANCH_NAME}-python${PYTHON_VERSION}-ci
HOST_USER_ID="$(id -ur)"
export HOST_USER_ID
HOST_GROUP_ID="$(id -gr)"
export HOST_GROUP_ID
docker-compose \
-f "${MY_DIR}/docker-compose/base.yml" \
-f "${MY_DIR}/docker-compose/local.yml" \
run airflow-testing /opt/airflow/scripts/ci/in_container/run_fix_ownership.sh
script_end

43
scripts/ci/ci_flake8.sh
Просмотреть файл

@ -15,24 +15,43 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -euo pipefail
MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
export AIRFLOW_CI_SILENT=${AIRFLOW_CI_SILENT:="true"}
export PYTHON_VERSION=${PYTHON_VERSION:-3.6}
# shellcheck source=scripts/ci/_utils.sh
. "${MY_DIR}/_utils.sh"
# shellcheck source=scripts/ci/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
basic_sanity_checks
function run_flake8() {
FILES=("$@")
if [[ "${#FILES[@]}" == "0" ]]; then
docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \
--entrypoint "/usr/local/bin/dumb-init" \
--env PYTHONDONTWRITEBYTECODE \
--env AIRFLOW_CI_VERBOSE="${VERBOSE}" \
--env AIRFLOW_CI_SILENT \
--env HOST_USER_ID="$(id -ur)" \
--env HOST_GROUP_ID="$(id -gr)" \
--rm \
"${AIRFLOW_CI_IMAGE}" \
"--" "/opt/airflow/scripts/ci/in_container/run_flake8.sh" \
| tee -a "${OUTPUT_LOG}"
else
docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \
--entrypoint "/usr/local/bin/dumb-init" \
--env PYTHONDONTWRITEBYTECODE \
--env AIRFLOW_CI_VERBOSE="${VERBOSE}" \
--env AIRFLOW_CI_SILENT \
--env HOST_USER_ID="$(id -ur)" \
--env HOST_GROUP_ID="$(id -gr)" \
--rm \
"${AIRFLOW_CI_IMAGE}" \
"--" "/opt/airflow/scripts/ci/in_container/run_flake8.sh" "${FILES[@]}" \
| tee -a "${OUTPUT_LOG}"
fi
}
script_start
rebuild_ci_image_if_needed
run_flake8 "$@"
script_end

42
scripts/ci/ci_lint_dockerfile.sh
Просмотреть файл

@ -15,20 +15,38 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -euo pipefail
MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
export AIRFLOW_CI_SILENT=${AIRFLOW_CI_SILENT:="true"}
# shellcheck source=scripts/ci/_utils.sh
. "${MY_DIR}/_utils.sh"
# shellcheck source=scripts/ci/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
basic_sanity_checks
script_start
function run_docker_lint() {
FILES=("$@")
if [[ "${#FILES[@]}" == "0" ]]; then
echo
echo "Running docker lint for all Dockerfiles"
echo
docker run \
-v "$(pwd):/root" \
-w /root \
--rm \
hadolint/hadolint /bin/hadolint Dockerfile*
echo
echo "Docker pylint completed with no errors"
echo
else
echo
echo "Running docker lint for $*"
echo
docker run \
-v "$(pwd):/root" \
-w /root \
--rm \
hadolint/hadolint /bin/hadolint "$@"
echo
echo "Docker pylint completed with no errors"
echo
fi
}
run_docker_lint "$@"
script_end

36
scripts/ci/ci_mypy.sh
Просмотреть файл

@ -15,24 +15,32 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -euo pipefail
MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
export AIRFLOW_CI_SILENT=${AIRFLOW_CI_SILENT:="true"}
export PYTHON_VERSION=${PYTHON_VERSION:-3.6}
# shellcheck source=scripts/ci/_utils.sh
. "${MY_DIR}/_utils.sh"
basic_sanity_checks
script_start
# shellcheck source=scripts/ci/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
rebuild_ci_image_if_needed
run_mypy "$@"
function run_mypy() {
FILES=("$@")
if [[ "${#FILES[@]}" == "0" ]]; then
FILES=(airflow tests docs)
fi
script_end
docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \
--entrypoint "/usr/local/bin/dumb-init" \
--env PYTHONDONTWRITEBYTECODE \
--env AIRFLOW_CI_VERBOSE="${VERBOSE}" \
--env AIRFLOW_CI_SILENT \
--env HOST_USER_ID="$(id -ur)" \
--env HOST_GROUP_ID="$(id -gr)" \
"-v" "${AIRFLOW_SOURCES}/.mypy_cache:/opt/airflow/.mypy_cache" \
--rm \
"${AIRFLOW_CI_IMAGE}" \
"--" "/opt/airflow/scripts/ci/in_container/run_mypy.sh" "${FILES[@]}" \
| tee -a "${OUTPUT_LOG}"
}
run_mypy "$@"

41
scripts/ci/ci_pylint_main.sh
Просмотреть файл

@ -15,21 +15,40 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -euo pipefail
MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
export AIRFLOW_CI_SILENT=${AIRFLOW_CI_SILENT:="true"}
export PYTHON_VERSION=${PYTHON_VERSION:-3.6}
# shellcheck source=scripts/ci/_utils.sh
. "${MY_DIR}/_utils.sh"
# shellcheck source=scripts/ci/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
basic_sanity_checks
script_start
function run_pylint_main() {
FILES=("$@")
if [[ "${#FILES[@]}" == "0" ]]; then
docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \
--entrypoint "/usr/local/bin/dumb-init" \
--env PYTHONDONTWRITEBYTECODE \
--env AIRFLOW_CI_VERBOSE="${VERBOSE}" \
--env AIRFLOW_CI_SILENT \
--env HOST_USER_ID="$(id -ur)" \
--env HOST_GROUP_ID="$(id -gr)" \
--rm \
"${AIRFLOW_CI_IMAGE}" \
"--" "/opt/airflow/scripts/ci/in_container/run_pylint_main.sh" \
| tee -a "${OUTPUT_LOG}"
else
docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \
--entrypoint "/usr/local/bin/dumb-init" \
--env PYTHONDONTWRITEBYTECODE \
--env AIRFLOW_CI_VERBOSE="${VERBOSE}" \
--env AIRFLOW_CI_SILENT \
--env HOST_USER_ID="$(id -ur)" \
--env HOST_GROUP_ID="$(id -gr)" \
--rm \
"${AIRFLOW_CI_IMAGE}" \
"--" "/opt/airflow/scripts/ci/in_container/run_pylint_main.sh" "${FILES[@]}" \
| tee -a "${OUTPUT_LOG}"
fi
}
rebuild_ci_image_if_needed

41
scripts/ci/ci_pylint_tests.sh
Просмотреть файл

@ -16,20 +16,41 @@
# specific language governing permissions and limitations
# under the License.
set -euo pipefail
MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
export AIRFLOW_CI_SILENT=${AIRFLOW_CI_SILENT:="true"}
export PYTHON_VERSION=${PYTHON_VERSION:-3.6}
# shellcheck source=scripts/ci/_utils.sh
. "${MY_DIR}/_utils.sh"
# shellcheck source=scripts/ci/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
basic_sanity_checks
function run_pylint_tests() {
FILES=("$@")
if [[ "${#FILES[@]}" == "0" ]]; then
docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \
--entrypoint "/usr/local/bin/dumb-init" \
--env PYTHONDONTWRITEBYTECODE \
--env AIRFLOW_CI_VERBOSE="${VERBOSE}" \
--env AIRFLOW_CI_SILENT \
--env HOST_USER_ID="$(id -ur)" \
--env HOST_GROUP_ID="$(id -gr)" \
--rm \
"${AIRFLOW_CI_IMAGE}" \
"--" "/opt/airflow/scripts/ci/in_container/run_pylint_tests.sh" \
| tee -a "${OUTPUT_LOG}"
else
docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \
--entrypoint "/usr/local/bin/dumb-init" \
--env PYTHONDONTWRITEBYTECODE \
--env AIRFLOW_CI_VERBOSE="${VERBOSE}" \
--env AIRFLOW_CI_SILENT \
--env HOST_USER_ID="$(id -ur)" \
--env HOST_GROUP_ID="$(id -gr)" \
--rm \
"${AIRFLOW_CI_IMAGE}" \
"--" "/opt/airflow/scripts/ci/in_container/run_pylint_tests.sh" "${FILES[@]}" \
| tee -a "${OUTPUT_LOG}"
fi
}
script_start
rebuild_ci_image_if_needed
@ -44,5 +65,3 @@ if [[ "${#@}" != "0" ]]; then
else
run_pylint_tests
fi
script_end

26
scripts/ci/ci_refresh_pylint_todo.sh
Просмотреть файл

@ -15,23 +15,25 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -euo pipefail
MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
export AIRFLOW_CI_SILENT=${AIRFLOW_CI_SILENT:="false"}
export FORCE_ANSWER_TO_QUESTIONS=quit
# shellcheck source=scripts/ci/_utils.sh
. "${MY_DIR}/_utils.sh"
# shellcheck source=scripts/ci/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
basic_sanity_checks
script_start
function refresh_pylint_todo() {
docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \
--env PYTHONDONTWRITEBYTECODE \
--env AIRFLOW_CI_VERBOSE="${VERBOSE}" \
--env AIRFLOW_CI_SILENT \
--env HOST_USER_ID="$(id -ur)" \
--env HOST_GROUP_ID="$(id -gr)" \
--rm \
"${AIRFLOW_CI_IMAGE}" \
/opt/airflow/scripts/ci/in_container/refresh_pylint_todo.sh \
| tee -a "${OUTPUT_LOG}"
}
rebuild_ci_image_if_needed
refresh_pylint_todo
script_end

20
scripts/ci/ci_run_airflow_testing.sh
Просмотреть файл

@ -15,18 +15,10 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -euo pipefail
MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
# shellcheck source=scripts/ci/_utils.sh
. "${MY_DIR}/_utils.sh"
export VERBOSE=${VERBOSE:="true"}
basic_sanity_checks
script_start
# shellcheck source=scripts/ci/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
if [[ -f ${BUILD_CACHE_DIR}/.skip_tests ]]; then
echo
@ -60,12 +52,6 @@ echo
echo "Using docker image: ${AIRFLOW_CI_IMAGE} for docker compose runs"
echo
HOST_USER_ID="$(id -ur)"
export HOST_USER_ID
HOST_GROUP_ID="$(id -gr)"
export HOST_GROUP_ID
INTEGRATIONS=()
ENABLED_INTEGRATIONS=${ENABLED_INTEGRATIONS:=""}
@ -111,5 +97,3 @@ else
# and bash -c starts passing parameters from $0. TODO: fixme
set -u
fi
script_end

16
scripts/ci/ci_run_all_static_checks.sh
Просмотреть файл

@ -15,21 +15,11 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -euo pipefail
MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
export AIRFLOW_CI_SILENT=${AIRFLOW_CI_SILENT:="true"}
export PYTHON_VERSION=${PYTHON_VERSION:-3.6}
# shellcheck source=scripts/ci/_utils.sh
. "${MY_DIR}/_utils.sh"
basic_sanity_checks
script_start
# shellcheck source=scripts/ci/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
if [[ -f ${BUILD_CACHE_DIR}/.skip_tests ]]; then
echo
@ -45,5 +35,3 @@ IMAGES_TO_CHECK=("CI")
export IMAGES_TO_CHECK
pre-commit run --all-files --show-diff-on-failure --verbose
script_end

16
scripts/ci/ci_run_static_checks_pylint_tests.sh
Просмотреть файл

@ -15,21 +15,11 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -euo pipefail
MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
export AIRFLOW_CI_SILENT=${AIRFLOW_CI_SILENT:="true"}
export PYTHON_VERSION=${PYTHON_VERSION:-3.6}
# shellcheck source=scripts/ci/_utils.sh
. "${MY_DIR}/_utils.sh"
basic_sanity_checks
script_start
# shellcheck source=scripts/ci/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
if [[ -f ${BUILD_CACHE_DIR}/.skip_tests ]]; then
echo
@ -45,5 +35,3 @@ IMAGES_TO_CHECK=("CI")
export IMAGES_TO_CHECK
pre-commit run pylint-tests --all-files --show-diff-on-failure --verbose
script_end

31
scripts/ci/in_container/_in_container_script_init.sh Executable file
Просмотреть файл

@ -0,0 +1,31 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -euo pipefail
# This should only be sourced from in_container directory!
MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
# shellcheck source=scripts/ci/in_container/_in_container_utils.sh
. "${MY_DIR}/_in_container_utils.sh"
in_container_basic_sanity_check
in_container_script_start
trap in_container_script_end EXIT

11
scripts/ci/in_container/_in_container_utils.sh
Просмотреть файл

@ -16,9 +16,6 @@
# specific language governing permissions and limitations
# under the License.
#
# Asserts that you are actually in container
#
function assert_in_container() {
if [[ ! -f /.dockerenv ]]; then
echo >&2
@ -38,6 +35,14 @@ function in_container_script_start() {
}
function in_container_script_end() {
#shellcheck disable=2181
EXIT_CODE=$?
if [[ ${EXIT_CODE} != 0 ]]; then
echo "###########################################################################################"
echo " EXITING ${0} WITH STATUS CODE ${EXIT_CODE}"
echo "###########################################################################################"
fi
if [[ ${AIRFLOW_CI_VERBOSE:="false"} == "true" ]]; then
set +x
fi

28
scripts/ci/in_container/check_environment.sh
Просмотреть файл

@ -15,30 +15,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# Script to check licences for all code. Can be started from any working directory
set -euo pipefail
MY_DIR=$(cd "$(dirname "$0")" || exit 1; pwd)
# shellcheck source=scripts/ci/in_container/_in_container_utils.sh
. "${MY_DIR}/_in_container_utils.sh"
in_container_basic_sanity_check
in_container_script_start
function on_exit() {
#shellcheck disable=2181
EXIT_CODE=$?
if [[ ${EXIT_CODE} != 0 ]]; then
echo "###########################################################################################"
echo " EXITING WITH STATUS CODE ${EXIT_CODE}"
echo "###########################################################################################"
fi
}
# shellcheck source=scripts/ci/in_container/_in_container_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/_in_container_script_init.sh"
export EXIT_CODE=0
export DISABLED_INTEGRATIONS=""
@ -91,7 +70,6 @@ function check_integration {
echo "-----------------------------------------------------------------------------------------------"
}
trap on_exit EXIT
echo
echo "Check CI environment sanity!"
echo
@ -226,5 +204,3 @@ fi
echo
echo "CI environment is sane!"
echo
in_container_script_end

15
scripts/ci/in_container/deploy_airflow_to_kubernetes.sh
Просмотреть файл

@ -15,20 +15,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# Script to run flake8 on all code. Can be started from any working directory
set -uo pipefail
MY_DIR=$(cd "$(dirname "$0")" || exit 1; pwd)
# shellcheck source=scripts/ci/in_container/_in_container_utils.sh
. "${MY_DIR}/_in_container_utils.sh"
in_container_basic_sanity_check
in_container_script_start
# shellcheck source=scripts/ci/in_container/_in_container_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/_in_container_script_init.sh"
"${MY_DIR}/kubernetes/docker/rebuild_airflow_image.sh"
"${MY_DIR}/kubernetes/app/deploy_app.sh"
in_container_script_end

17
scripts/ci/in_container/entrypoint_ci.sh
Просмотреть файл

@ -15,23 +15,12 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# Bash sanity settings (error on exit, complain for undefined vars, error when pipe fails)
set -euo pipefail
MY_DIR=$(cd "$(dirname "$0")" || exit 1; pwd)
if [[ ${AIRFLOW_CI_VERBOSE:="false"} == "true" ]]; then
set -x
fi
# shellcheck source=scripts/ci/in_container/_in_container_utils.sh
. "${MY_DIR}/_in_container_utils.sh"
in_container_basic_sanity_check
in_container_script_start
# shellcheck source=scripts/ci/in_container/_in_container_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/_in_container_script_init.sh"
TRAVIS=${TRAVIS:=}
@ -226,5 +215,3 @@ fi
ARGS=("${CI_ARGS[@]}" "${TESTS_TO_RUN}")
"${MY_DIR}/run_ci_tests.sh" "${ARGS[@]}"
in_container_script_end

18
scripts/ci/in_container/kubernetes/app/deploy_app.sh
Просмотреть файл

@ -16,7 +16,8 @@
# specific language governing permissions and limitations
# under the License.
set -eu
set -euo pipefail
MY_DIR=$(cd "$(dirname "$0")" && pwd)
AIRFLOW_SOURCES=$(
@ -25,6 +26,7 @@ AIRFLOW_SOURCES=$(
)
export AIRFLOW_SOURCES
# We keep _utils here because we are not in the in_container directory
# shellcheck source=scripts/ci/in_container/_in_container_utils.sh
. "${MY_DIR}/../../_in_container_utils.sh"
@ -32,6 +34,13 @@ assert_in_container
in_container_script_start
function end_and_dump_logs() {
dump_logs
in_container_script_end
}
trap in_container_script_end EXIT
export TEMPLATE_DIRNAME="${MY_DIR}/templates"
export BUILD_DIRNAME="${MY_DIR}/build"
@ -134,6 +143,7 @@ dump_logs() {
}
set +x
set +o pipefail
# wait for up to 10 minutes for everything to be deployed
PODS_ARE_READY="0"
for i in {1..150}; do
@ -178,15 +188,11 @@ for i in {1..30}; do
fi
sleep 10
done
set -o pipefail
if [[ "${AIRFLOW_WEBSERVER_IS_READY}" == "1" ]]; then
echo "Airflow webserver is ready."
else
echo >&2 "Airflow webserver is not ready after waiting for a long time. Exiting..."
dump_logs
exit 1
fi
dump_logs
in_container_script_end

4
scripts/ci/in_container/kubernetes/docker/rebuild_airflow_image.sh
Просмотреть файл

@ -15,11 +15,11 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -euo pipefail
MY_DIR=$(cd "$(dirname "$0")" && pwd)
AIRFLOW_SOURCES=$(cd "${MY_DIR}/../../../../../" || exit 1 ; pwd)
export AIRFLOW_SOURCES
# We keep _utils here because we are not in the in_container directory
# shellcheck source=scripts/ci/in_container/_in_container_utils.sh
. "${MY_DIR}/../../_in_container_utils.sh"
@ -29,7 +29,7 @@ assert_in_container
in_container_script_start
cd "${AIRFLOW_SOURCES}"
cd "${AIRFLOW_SOURCES}" || exit 1
# Required to rebuild images from inside container
mkdir -pv scripts/docker/

3
scripts/ci/in_container/kubernetes/setup_kind_cluster.sh
Просмотреть файл

@ -15,8 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -euo pipefail
MY_DIR=$(cd "$(dirname "$0")" && pwd)
AIRFLOW_SOURCES=$(
@ -27,6 +25,7 @@ export AIRFLOW_SOURCES
export RECREATE_KIND_CLUSTER=${RECREATE_KIND_CLUSTER:="true"}
# We keep _utils here because we are not in the in_container directory
# shellcheck source=scripts/ci/in_container/_in_container_utils.sh
. "${MY_DIR}/../_in_container_utils.sh"

16
scripts/ci/in_container/refresh_pylint_todo.sh
Просмотреть файл

@ -15,19 +15,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# Script to run Pylint on main code. Can be started from any working directory
set -uo pipefail
MY_DIR=$(cd "$(dirname "$0")" || exit 1; pwd)
# shellcheck source=scripts/ci/in_container/_in_container_utils.sh
. "${MY_DIR}/_in_container_utils.sh"
in_container_basic_sanity_check
in_container_script_start
# shellcheck source=scripts/ci/in_container/_in_container_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/_in_container_script_init.sh"
in_container_refresh_pylint_todo
in_container_script_end

19
scripts/ci/in_container/run_check_licence.sh
Просмотреть файл

@ -15,18 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# Script to check licences for all code. Can be started from any working directory
set -uo pipefail
MY_DIR=$(cd "$(dirname "$0")" || exit 1; pwd)
# shellcheck source=scripts/ci/in_container/_in_container_utils.sh
. "${MY_DIR}/_in_container_utils.sh"
in_container_basic_sanity_check
in_container_script_start
# shellcheck source=scripts/ci/in_container/_in_container_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/_in_container_script_init.sh"
echo
echo "Running Licence check"
@ -44,9 +34,9 @@ if ! java -jar "/opt/apache-rat.jar" -E "${AIRFLOW_SOURCES}"/.rat-excludes \
exit 1
fi
set +e
ERRORS=$(grep -e "??" "${AIRFLOW_SOURCES}/logs/rat-results.txt")
in_container_script_end
set -e
in_container_fix_ownership
@ -59,4 +49,5 @@ if test ! -z "${ERRORS}"; then
else
echo "RAT checks passed."
echo
exit 0
fi

18
scripts/ci/in_container/run_ci_tests.sh
Просмотреть файл

@ -15,19 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# Bash sanity settings (error on exit, complain for undefined vars, error when pipe fails)
set -euo pipefail
MY_DIR=$(cd "$(dirname "$0")" || exit 1; pwd)
# shellcheck source=scripts/ci/in_container/_in_container_utils.sh
. "${MY_DIR}/_in_container_utils.sh"
in_container_basic_sanity_check
in_container_script_start
# shellcheck source=scripts/ci/in_container/_in_container_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/_in_container_script_init.sh"
# any argument received is overriding the default nose execution arguments:
PYTEST_ARGS=( "$@" )
@ -55,7 +44,4 @@ if [[ ${CI} == "true" && ${ENABLE_KIND_CLUSTER} == "true" ]]; then
send_kubernetes_logs_to_file_io
fi
in_container_script_end
exit "${RES}"

16
scripts/ci/in_container/run_docs_build.sh
Просмотреть файл

@ -15,23 +15,11 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# Script to build docs. Can be started from any working directory
set -uo pipefail
MY_DIR=$(cd "$(dirname "$0")" || exit 1; pwd)
# shellcheck source=scripts/ci/in_container/_in_container_utils.sh
. "${MY_DIR}/_in_container_utils.sh"
in_container_basic_sanity_check
in_container_script_start
# shellcheck source=scripts/ci/in_container/_in_container_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/_in_container_script_init.sh"
sudo rm -rf "$(pwd)/docs/_build/*"
sudo rm -rf "$(pwd)/docs/_api/*"
sudo -E "$(pwd)/docs/build.sh"
in_container_fix_ownership
in_container_script_end

17
scripts/ci/in_container/run_extract_tests.sh
Просмотреть файл

@ -15,24 +15,11 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# Bash sanity settings (error on exit, complain for undefined vars, error when pipe fails)
set -euo pipefail
MY_DIR=$(cd "$(dirname "$0")" || exit 1; pwd)
# shellcheck source=scripts/ci/in_container/_in_container_utils.sh
. "${MY_DIR}/_in_container_utils.sh"
in_container_basic_sanity_check
in_container_script_start
# shellcheck source=scripts/ci/in_container/_in_container_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/_in_container_script_init.sh"
TMP_FILE=$(mktemp)
nosetests --collect-only --with-xunit --xunit-file="${TMP_FILE}"
python "${AIRFLOW_SOURCES}/tests/test_utils/get_all_tests.py" "${TMP_FILE}" | sort >> "${HOME}/all_tests.txt"
in_container_script_end

15
scripts/ci/in_container/run_fix_ownership.sh
Просмотреть файл

@ -15,18 +15,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# Bash sanity settings (error on exit, complain for undefined vars, error when pipe fails)
set -euo pipefail
MY_DIR=$(cd "$(dirname "$0")" || exit 1; pwd)
# shellcheck source=scripts/ci/in_container/_in_container_utils.sh
. "${MY_DIR}/_in_container_utils.sh"
in_container_basic_sanity_check
in_container_script_start
# shellcheck source=scripts/ci/in_container/_in_container_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/_in_container_script_init.sh"
in_container_fix_ownership
in_container_script_end

18
scripts/ci/in_container/run_flake8.sh
Просмотреть файл

@ -15,18 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# Script to run flake8 on all code. Can be started from any working directory
set -uo pipefail
MY_DIR=$(cd "$(dirname "$0")" || exit 1; pwd)
# shellcheck source=scripts/ci/in_container/_in_container_utils.sh
. "${MY_DIR}/_in_container_utils.sh"
in_container_basic_sanity_check
in_container_script_start
# shellcheck source=scripts/ci/in_container/_in_container_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/_in_container_script_init.sh"
if [[ ${#@} == "0" ]]; then
print_in_container_info
@ -38,11 +28,13 @@ else
print_in_container_info
fi
set +e
flake8 "$@"
RES="$?"
in_container_script_end
set -e
if [[ "${RES}" != 0 ]]; then
echo >&2

18
scripts/ci/in_container/run_mypy.sh
Просмотреть файл

@ -15,29 +15,21 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# Script to run mypy on all code. Can be started from any working directory
set -uo pipefail
MY_DIR=$(cd "$(dirname "$0")" || exit 1; pwd)
# shellcheck source=scripts/ci/in_container/_in_container_utils.sh
. "${MY_DIR}/_in_container_utils.sh"
in_container_basic_sanity_check
in_container_script_start
# shellcheck source=scripts/ci/in_container/_in_container_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/_in_container_script_init.sh"
print_in_container_info
print_in_container_info "Running mypy with parameters: $*"
print_in_container_info
print_in_container_info
mypy "$@"
set +e
mypy "$@"
RES="$?"
in_container_script_end
set -e
if [[ "${RES}" != 0 ]]; then
echo >&2

19
scripts/ci/in_container/run_pylint_main.sh
Просмотреть файл

@ -15,21 +15,13 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# Script to run Pylint on main code. Can be started from any working directory
set -uo pipefail
MY_DIR=$(cd "$(dirname "$0")" || exit 1; pwd)
# shellcheck source=scripts/ci/in_container/_in_container_utils.sh
. "${MY_DIR}/_in_container_utils.sh"
in_container_basic_sanity_check
in_container_script_start
# shellcheck source=scripts/ci/in_container/_in_container_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/_in_container_script_init.sh"
export PYTHONPATH=${AIRFLOW_SOURCES}
set +e
if [[ ${#@} == "0" ]]; then
echo
echo "Running pylint for all sources except 'tests' folder"
@ -61,8 +53,7 @@ else
/usr/local/bin/pylint --output-format=colorized "$@"
RES=$?
fi
in_container_script_end
set -e
if [[ "${RES}" != 0 ]]; then
echo >&2

18
scripts/ci/in_container/run_pylint_tests.sh
Просмотреть файл

@ -15,21 +15,13 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# Script to run Pylint on test code. Can be started from any working directory
set -uo pipefail
MY_DIR=$(cd "$(dirname "$0")" || exit 1; pwd)
# shellcheck source=scripts/ci/in_container/_in_container_utils.sh
. "${MY_DIR}/_in_container_utils.sh"
in_container_basic_sanity_check
in_container_script_start
# shellcheck source=scripts/ci/in_container/_in_container_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/_in_container_script_init.sh"
export PYTHONPATH=${AIRFLOW_SOURCES}
set +e
if [[ ${#@} == "0" ]]; then
echo
echo "Running pylint for 'tests' folder"
@ -48,7 +40,7 @@ else
RES=$?
fi
in_container_script_end
set -e
if [[ "${RES}" != 0 ]]; then
echo >&2

7
scripts/ci/pre_commit_bat_tests.sh
Просмотреть файл

@ -15,10 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -uo pipefail
MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
export FORCE_ANSWER_TO_QUESTIONS=${FORCE_ANSWER_TO_QUESTIONS:="quit"}
export REMEMBER_LAST_ANSWER="true"
@ -28,4 +24,5 @@ else
PARAMS=("${@}")
fi
"${MY_DIR}/ci_bat_tests.sh" "${PARAMS[@]}"
# shellcheck source=scripts/ci/ci_bat_tests.sh
. "$( dirname "${BASH_SOURCE[0]}" )/ci_bat_tests.sh" "${PARAMS[@]}"

2
scripts/ci/pre_commit_breeze_cmd_line.sh
Просмотреть файл

@ -16,7 +16,7 @@
# specific language governing permissions and limitations
# under the License.
set -uo pipefail
set -euo pipefail
MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

8
scripts/ci/pre_commit_check_license.sh
Просмотреть файл

@ -15,13 +15,11 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -euo pipefail
set -uo pipefail
MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
export FORCE_ANSWER_TO_QUESTIONS=${FORCE_ANSWER_TO_QUESTIONS:="quit"}
export REMEMBER_LAST_ANSWER="true"
# Hide lines between ****/**** (detailed list of files)
"${MY_DIR}/ci_check_license.sh" 2>&1 | \
sed "/Files with Apache License headers will be marked AL.*$/,/^\**$/d"
"$( dirname "${BASH_SOURCE[0]}" )/ci_check_license.sh" 2>&1 | \
(sed "/Files with Apache License headers will be marked AL.*$/,/^\**$/d" || true)

14
scripts/ci/pre_commit_ci_build.sh
Просмотреть файл

@ -15,20 +15,10 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -uo pipefail
MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
export REMEMBER_LAST_ANSWER="false"
export PYTHON_VERSION=${PYTHON_VERSION:-3.6}
# shellcheck source=scripts/ci/_utils.sh
. "${MY_DIR}/_utils.sh"
basic_sanity_checks
script_start
# shellcheck source=scripts/ci/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
rebuild_all_images_if_needed_and_confirmed
script_end

7
scripts/ci/pre_commit_flake8.sh
Просмотреть файл

@ -15,11 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -uo pipefail
MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
export FORCE_ANSWER_TO_QUESTIONS=${FORCE_ANSWER_TO_QUESTIONS:="quit"}
export REMEMBER_LAST_ANSWER="true"
"${MY_DIR}/ci_flake8.sh" "${@}"
# shellcheck source=scripts/ci/ci_flake8.sh
. "$( dirname "${BASH_SOURCE[0]}" )/ci_flake8.sh" "${@}"

7
scripts/ci/pre_commit_lint_dockerfile.sh
Просмотреть файл

@ -15,10 +15,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -uo pipefail
MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
export REMEMBER_LAST_ANSWER="true"
"${MY_DIR}/ci_lint_dockerfile.sh" "${@}"
# shellcheck source=scripts/ci/ci_lint_dockerfile.sh
. "$( dirname "${BASH_SOURCE[0]}" )/ci_lint_dockerfile.sh" "${@}"

7
scripts/ci/pre_commit_mypy.sh
Просмотреть файл

@ -15,11 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -uo pipefail
MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
export FORCE_ANSWER_TO_QUESTIONS=${FORCE_ANSWER_TO_QUESTIONS:="quit"}
export REMEMBER_LAST_ANSWER="true"
"${MY_DIR}/ci_mypy.sh" "${@}"
# shellcheck source=scripts/ci/ci_mypy.sh
. "$( dirname "${BASH_SOURCE[0]}" )/ci_mypy.sh" "${@}"

7
scripts/ci/pre_commit_pylint_main.sh
Просмотреть файл

@ -15,11 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -uo pipefail
MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
export FORCE_ANSWER_TO_QUESTIONS=${FORCE_ANSWER_TO_QUESTIONS:="quit"}
export REMEMBER_LAST_ANSWER="true"
"${MY_DIR}/ci_pylint_main.sh" "${@}"
# shellcheck source=scripts/ci/ci_pylint_main.sh
. "$( dirname "${BASH_SOURCE[0]}" )/ci_pylint_main.sh" "${@}"

7
scripts/ci/pre_commit_pylint_tests.sh
Просмотреть файл

@ -15,11 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -uo pipefail
MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
export FORCE_ANSWER_TO_QUESTIONS=${FORCE_ANSWER_TO_QUESTIONS:="quit"}
export REMEMBER_LAST_ANSWER="true"
"${MY_DIR}/ci_pylint_tests.sh" "${@}"
# shellcheck source=scripts/ci/ci_pylint_tests.sh
. "$( dirname "${BASH_SOURCE[0]}" )/ci_pylint_tests.sh" "${@}"

13
tests/bats/test_breeze_params.bats
Просмотреть файл

@ -45,6 +45,8 @@ EOF
@test "Test wrong value for a parameter but proper stored in the .build/PARAM" {
load bats_utils
initialize_breeze_environment
export _BREEZE_ALLOWED_TEST_PARAMS=" a b c "
export TEST_PARAM=x
echo "a" > "${AIRFLOW_SOURCES}/.build/.TEST_PARAM"
@ -63,6 +65,8 @@ EOF
@test "Test wrong value for a parameter stored in the .build/PARAM" {
load bats_utils
initialize_breeze_environment
export _BREEZE_ALLOWED_TEST_PARAMS=" a b c "
export TEST_PARAM=x
echo "x" > "${AIRFLOW_SOURCES}/.build/.TEST_PARAM"
@ -82,6 +86,9 @@ EOF
@test "Test correct value for a parameter" {
load bats_utils
initialize_breeze_environment
export _BREEZE_ALLOWED_TEST_PARAMS=" a b c "
export TEST_PARAM=a
run check_for_allowed_params "TEST_PARAM" "Test Param" "--message"
@ -93,6 +100,9 @@ EOF
@test "Test read_parameter from missing file" {
load bats_utils
initialize_breeze_environment
run read_from_file TEST_PARAM
[ -z "${TEST_FILE}" ]
diff <(echo "${output}") <(echo "")
@ -102,6 +112,9 @@ EOF
@test "Test read_parameter from file" {
load bats_utils
initialize_breeze_environment
echo "a" > "${AIRFLOW_SOURCES}/.build/.TEST_PARAM"
run read_from_file TEST_PARAM
diff <(echo "${output}") <(echo "a")

3
tests/bats/test_empty_test.bats
Просмотреть файл

@ -19,6 +19,9 @@
@test "empty test" {
load bats_utils
initialize_breeze_environment
run pwd
[ "${status}" == 0 ]
}

9
tests/bats/test_yaml_parser.bats
Просмотреть файл

@ -20,6 +20,9 @@
@test "yaml parsing missing file" {
load bats_utils
initialize_breeze_environment
run parse_yaml
[ "${status}" == 1 ]
[ "${output}" == "Please provide yaml filename as first parameter." ]
@ -27,6 +30,9 @@
@test "yaml parsing existing file" {
load bats_utils
initialize_breeze_environment
run parse_yaml "scripts/ci/docker-compose/local.yml"
diff <(echo "${output}") - << 'EOF'
1="--"
@ -71,6 +77,9 @@ EOF
@test "convert yaml docker file to docker params" {
load bats_utils
initialize_breeze_environment
run convert_docker_mounts_to_docker_params
diff <(echo "${output}") - << EOF
-v