From ef71ac6a2240ad1ef55b9ca810ea85f5b62c09ea Mon Sep 17 00:00:00 2001 From: Ash Berlin-Taylor Date: Fri, 13 Mar 2020 18:54:22 +0000 Subject: [PATCH] [AIRFLOW-7029] Use separate docker image for running license check (#7678) Each stage of the CI tests needs to pull our `ci` image. By removing java from it we can save 1-2minutes from each test stage. This is part of that work. --- Dockerfile | 39 -------------- scripts/ci/ci_check_license.sh | 46 +++++++++++------ scripts/ci/in_container/run_check_licence.sh | 53 -------------------- 3 files changed, 32 insertions(+), 106 deletions(-) delete mode 100755 scripts/ci/in_container/run_check_licence.sh diff --git a/Dockerfile b/Dockerfile index 57f8233292..39f0c02155 100644 --- a/Dockerfile +++ b/Dockerfile @@ -243,45 +243,6 @@ RUN KIND_URL="https://github.com/kubernetes-sigs/kind/releases/download/${KIND_V && curl --fail --location "${KIND_URL}" --output "/usr/local/bin/kind" \ && chmod +x /usr/local/bin/kind -# Install Apache RAT -ARG RAT_VERSION="0.13" -ARG RAT_BACKUP_SITE_1="https://www-eu.apache.org/dist/creadur" -ARG RAT_BACKUP_SITE_2="https://www-us.apache.org/dist/creadur" -# It's OK to use HTTP rather than https here as we verify it later with gpg from the -# offcial backup servers of Apache! -ARG RAT_MIRROR_1="http://mirror.serverion.com/apache/creadur" -ARG RAT_MIRROR_2="http://mirror.cc.columbia.edu/pub/software/apache/creadur" - -RUN RAT_TARGZ_FILE_NAME="apache-rat-${RAT_VERSION}-bin.tar.gz" \ - && RAT_FOLDER="apache-rat-${RAT_VERSION}" \ - && RAT_KEYS_URL_1="${RAT_BACKUP_SITE_1}/KEYS" \ - && RAT_KEYS_URL_2="${RAT_BACKUP_SITE_2}/KEYS" \ - && RAT_ASC_URL_1="${RAT_BACKUP_SITE_1}/${RAT_FOLDER}/${RAT_TARGZ_FILE_NAME}.asc" \ - && RAT_ASC_URL_2="${RAT_BACKUP_SITE_2}/${RAT_FOLDER}/${RAT_TARGZ_FILE_NAME}.asc" \ - && RAT_URL_1="${RAT_MIRROR_1}/${RAT_FOLDER}/${RAT_TARGZ_FILE_NAME}" \ - && RAT_URL_2="${RAT_MIRROR_2}/${RAT_FOLDER}/${RAT_TARGZ_FILE_NAME}" \ - && RAT_TAR_GZ="/opt/${RAT_TARGZ_FILE_NAME}" \ - && RAT_TAR_GZ_ASC="/opt/${RAT_TARGZ_FILE_NAME}.asc" \ - && RAT_KEYS="/opt/KEYS" \ - && RAT_JAR_IN_TAR="${RAT_FOLDER}/apache-rat-${RAT_VERSION}.jar" \ - && RAT_JAR="/opt/apache-rat.jar" \ - && echo "Downloading KEYS from backup Apache servers: ${RAT_KEYS_URL_1}, ${RAT_KEYS_URL_2}" \ - && (curl --fail --location "${RAT_KEYS_URL_1}" --output "${RAT_KEYS}" || \ - curl --fail --location "${RAT_KEYS_URL_2}" --output "${RAT_KEYS}") \ - && echo "Downloading ASC from backup Apache servers: ${RAT_ASC_URL_1}, ${RAT_ASC_URL_2}" \ - && (curl --fail --location "${RAT_ASC_URL_1}" --output "${RAT_TAR_GZ_ASC}" || \ - curl --fail --location "${RAT_ASC_URL_2}" --output "${RAT_TAR_GZ_ASC}") \ - && echo "Downloading RAT from mirrors: ${RAT_URL_1}, ${RAT_URL_2} to ${RAT_JAR}" \ - && (curl --fail --location "${RAT_URL_1}" --output "${RAT_TAR_GZ}" || \ - curl --fail --location "${RAT_URL_2}" --output "${RAT_TAR_GZ}") \ - && gpg --import ${RAT_KEYS} \ - && gpg --verify "${RAT_TAR_GZ_ASC}" "${RAT_TAR_GZ}" \ - && tar --extract --gzip --file "${RAT_TAR_GZ}" -C /opt "${RAT_JAR_IN_TAR}" \ - && mv -v /opt/"${RAT_JAR_IN_TAR}" "${RAT_JAR}" \ - && rm -vrf "${RAT_TAR_GZ}" "/opt/${RAT_FOLDER}" \ - && rm -f "${RAT_KEYS}" \ - && jar -tf "${RAT_JAR}" >/dev/null - # Setup PIP # By default PIP install run without cache to make image smaller ARG PIP_NO_CACHE_DIR="true" diff --git a/scripts/ci/ci_check_license.sh b/scripts/ci/ci_check_license.sh index b90f5312f0..301ef07254 100755 --- a/scripts/ci/ci_check_license.sh +++ b/scripts/ci/ci_check_license.sh @@ -22,21 +22,39 @@ export PYTHON_VERSION=${PYTHON_VERSION:-3.6} . "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" function run_check_license() { - docker run "${EXTRA_DOCKER_FLAGS[@]}" -t \ - --entrypoint "/usr/local/bin/dumb-init" \ - --env PYTHONDONTWRITEBYTECODE \ - --env VERBOSE \ - --env VERBOSE_COMMANDS \ - --env HOST_USER_ID="$(id -ur)" \ - --env HOST_GROUP_ID="$(id -gr)" \ + echo + echo "Running Licence check" + echo + + # This is the target of a symlink in airflow/www/static/docs - + # and rat exclude doesn't cope with the symlink target doesn't exist + mkdir -p docs/_build/html/ + + echo "Running license checks. This can take a while." + + if ! docker run "${EXTRA_DOCKER_FLAGS[@]}" -t \ + --user "$(id -ur):$(id -gr)" \ --rm \ - "${AIRFLOW_CI_IMAGE}" \ - "--" "/opt/airflow/scripts/ci/in_container/run_check_licence.sh" \ - | tee -a "${OUTPUT_LOG}" + ashb/apache-rat:0.13-1 \ + --exclude-file /opt/airflow/.rat-excludes \ + --d /opt/airflow | tee "${AIRFLOW_SOURCES}/logs/rat-results.txt" ; then + echo >&2 "RAT exited abnormally" + exit 1 + fi + + set +e + ERRORS=$(grep -F "??" "${AIRFLOW_SOURCES}/logs/rat-results.txt") + set -e + if test ! -z "${ERRORS}"; then + echo >&2 + echo >&2 "Could not find Apache license headers in the following files:" + echo >&2 "${ERRORS}" + exit 1 + echo >&2 + else + echo "RAT checks passed." + echo + fi } -prepare_build - -rebuild_ci_image_if_needed - run_check_license diff --git a/scripts/ci/in_container/run_check_licence.sh b/scripts/ci/in_container/run_check_licence.sh deleted file mode 100755 index e207a10491..0000000000 --- a/scripts/ci/in_container/run_check_licence.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# shellcheck source=scripts/ci/in_container/_in_container_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_in_container_script_init.sh" - -echo -echo "Running Licence check" -echo - -# This is the target of a symlink in airflow/www/static/docs - -# and rat exclude doesn't cope with the symlink target doesn't exist -sudo mkdir -p docs/_build/html/ - -echo "Running license checks. This can take a while." - -if ! java -jar "/opt/apache-rat.jar" -E "${AIRFLOW_SOURCES}"/.rat-excludes \ - -d "${AIRFLOW_SOURCES}" | tee "${AIRFLOW_SOURCES}/logs/rat-results.txt" ; then - echo >&2 "RAT exited abnormally" - exit 1 -fi - -set +e -ERRORS=$(grep -e "??" "${AIRFLOW_SOURCES}/logs/rat-results.txt") -set -e - -in_container_fix_ownership - -if test ! -z "${ERRORS}"; then - echo >&2 - echo >&2 "Could not find Apache license headers in the following files:" - echo >&2 "${ERRORS}" - exit 1 - echo >&2 -else - echo "RAT checks passed." - echo - exit 0 -fi