[AIRFLOW-5585] Remove docker context from build

common/_files_for_rebuild_check.sh

@@ -1,3 +1,5 @@
+#!/usr/bin/env bash
+
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -15,11 +17,13 @@
 # specific language governing permissions and limitations
 # under the License.
 
-# We use this small Dockerfile to find out the list of files that are part of the Docker context
-# i.e. ignored by .dockerignore
-# We need it to fix permissions of files checked out by git to help with cache invalidation on different
-# system that have different UMASK. See hooks/build for some detailed explanation.
-
-FROM alpine:3.9
-
-COPY . /context
+FILES_FOR_REBUILD_CHECK=(
+ "setup.py"
+ "setup.cfg"
+ "Dockerfile"
+ "Dockerfile-checklicence"
+ ".dockerignore"
+ "airflow/version.py"
+ "airflow/www/package.json"
+ "airflow/www/package-lock.json" )
+export FILES_FOR_REBUILD_CHECK

hooks/build

@@ -31,6 +31,8 @@ cd "${AIRFLOW_SOURCES}"
 
 # shellcheck source=common/_default_branch.sh
 source "${AIRFLOW_SOURCES}/common/_default_branch.sh"
+# shellcheck source=common/_files_for_rebuild_check.sh
+source "${AIRFLOW_SOURCES}/common/_files_for_rebuild_check.sh"
 
 echo
 echo "Airflow root directory: ${AIRFLOW_SOURCES}"
@@ -446,7 +448,7 @@ DEPLOY_DIR=${AIRFLOW_SOURCES}/dist/${AIRFLOW_CONTAINER_BRANCH_NAME}/$(date +%Y-%
 mkdir -pv "${DEPLOY_DIR}"
 
 #
-# Fixing permissions for all filex that are going to be added to Docker context
+# Fixing permissions for all important files that are going to be added to Docker context
 # This is necessary, because there are different default umask settings on different *NIX
 # In case of some systems (especially in the CI environments) there is default +w group permission
 # set automatically via UMASK when git checkout is performed.
@@ -458,50 +460,29 @@ mkdir -pv "${DEPLOY_DIR}"
 #
 # We fix it by removing write permissions for other/group for all files that are in the Docker context.
 #
-if  [[ "${AIRFLOW_FIX_PERMISSIONS}" == "all" || "${AIRFLOW_FIX_PERMISSIONS}" == "setup" ]]; then
-    start_step "Fixing permissions for CI builds for ${AIRFLOW_FIX_PERMISSIONS} files"
-    if [[ "${AIRFLOW_FIX_PERMISSIONS}" == "all" ]]; then
-        # Get all files in the context - by building a small alpine based image
-        # then COPY all files (.) from the context and listing the files via find method
-        docker build -t airflow-context .  -f Dockerfile-context >/dev/null
-        ALL_FILES_TO_FIX=$(docker run --rm airflow-context /bin/sh -c "(cd /context && find .)")
-    elif [[ "${AIRFLOW_FIX_PERMISSIONS}" == "setup" ]]; then
-        ALL_FILES_TO_FIX="\
-            ${AIRFLOW_SOURCES}/setup.py \
-            ${AIRFLOW_SOURCES}/setup.cfg \
-            ${AIRFLOW_SOURCES}/airflow/version.py \
-            ${AIRFLOW_SOURCES}/airflow/__init__.py \
-            ${AIRFLOW_SOURCES}/airflow/bin/airflow \
-            ${AIRFLOW_SOURCES}/airflow/www/package.json \
-            ${AIRFLOW_SOURCES}/airflow/www/package-lock.json \
-            "
-    fi
-    STAT_BIN=stat
-    if [[ "${OSTYPE}" == "darwin"* ]]; then
-        STAT_BIN=gstat
-    fi
-
-    for FILE in ${ALL_FILES_TO_FIX}; do
-        ACCESS_RIGHTS=$("${STAT_BIN}" -c "%A" "${FILE}" || echo "--------")
-        # check if the file is group/other writeable
-        if [[ "${ACCESS_RIGHTS:5:1}" != "-" || "${ACCESS_RIGHTS:8:1}" != "-" ]]; then
-            if [[ "${VERBOSE_FIX_FILE:="false"}" == "true" ]]; then
-                "${STAT_BIN}" --printf "%a %A %F \t%s \t->    " "${FILE}"
-            fi
-            chmod og-w "${FILE}"
-            if [[ "${VERBOSE_FIX_FILE:="false"}" == "true" ]]; then
-                "${STAT_BIN}" --printf "%a %A %F \t%s \t%n\n" "${FILE}"
-            fi
-        fi
-    done
-
-    echo "Group/other write access removed for all $(echo "${ALL_FILES_TO_FIX}" | wc -w) files in context"
-    echo
-    echo
-else
-    echo "Skipping fixing permissions for CI builds"
+STAT_BIN=stat
+if [[ "${OSTYPE}" == "darwin"* ]]; then
+    STAT_BIN=gstat
 fi
 
+for FILE in "${FILES_FOR_REBUILD_CHECK[@]}"; do
+    ACCESS_RIGHTS=$("${STAT_BIN}" -c "%A" "${AIRFLOW_SOURCES}/${FILE}" || echo "--------")
+    # check if the file is group/other writeable
+    if [[ "${ACCESS_RIGHTS:5:1}" != "-" || "${ACCESS_RIGHTS:8:1}" != "-" ]]; then
+        if [[ "${VERBOSE_FIX_FILE:="false"}" == "true" ]]; then
+            "${STAT_BIN}" --printf "%a %A %F \t%s \t->    " "${AIRFLOW_SOURCES}/${FILE}"
+        fi
+        chmod og-w "${AIRFLOW_SOURCES}/${FILE}"
+        if [[ "${VERBOSE_FIX_FILE:="false"}" == "true" ]]; then
+            "${STAT_BIN}" --printf "%a %A %F \t%s \t%n\n" "${AIRFLOW_SOURCES}/${FILE}"
+        fi
+    fi
+done
+
+echo "Group/other write access removed for ${FILES_FOR_REBUILD_CHECK[*]}"
+echo
+echo
+
 start_step "Build Airflow CI slim image"
 
 if [[ "${AIRFLOW_CONTAINER_SKIP_SLIM_CI_IMAGE}" == "true" ]]; then

scripts/ci/_utils.sh

@@ -36,21 +36,14 @@ LAST_FORCE_ANSWER_FILE="${BUILD_CACHE_DIR}/last_force_answer.sh"
 IMAGES_TO_CHECK=("SLIM_CI" "CI" "CHECKLICENCE")
 export IMAGES_TO_CHECK
 
-FILES_FOR_REBUILD_CHECK="\
-setup.py \
-setup.cfg \
-Dockerfile \
-Dockerfile-checklicence \
-.dockerignore \
-airflow/version.py
-"
-
 mkdir -p "${AIRFLOW_SOURCES}/.mypy_cache"
 mkdir -p "${AIRFLOW_SOURCES}/logs"
 mkdir -p "${AIRFLOW_SOURCES}/tmp"
 
 # shellcheck source=common/_autodetect_variables.sh
 . "${AIRFLOW_SOURCES}/common/_autodetect_variables.sh"
+# shellcheck source=common/_files_for_rebuild_check.sh
+. "${AIRFLOW_SOURCES}/common/_files_for_rebuild_check.sh"
 
 # Default branch name for triggered builds is the one configured in default branch
 export AIRFLOW_CONTAINER_BRANCH_NAME=${AIRFLOW_CONTAINER_BRANCH_NAME:=${DEFAULT_BRANCH}}
@@ -205,7 +198,7 @@ function update_all_md5_files() {
     print_info
     print_info "Updating md5sum files"
     print_info
-    for FILE in ${FILES_FOR_REBUILD_CHECK}
+    for FILE in "${FILES_FOR_REBUILD_CHECK[@]}"
     do
         move_file_md5sum "${AIRFLOW_SOURCES}/${FILE}"
     done
@@ -247,7 +240,7 @@ function check_if_docker_build_is_needed() {
     if [[ ${AIRFLOW_CONTAINER_FORCE_DOCKER_BUILD:=""} == "true" ]]; then
         print_info "Docker image build is forced for ${THE_IMAGE_TYPE} image"
         set +e
-        for FILE in ${FILES_FOR_REBUILD_CHECK}
+        for FILE in "${FILES_FOR_REBUILD_CHECK[@]}"
         do
             # Just store md5sum for all files in md5sum.new - do not check if it is different
             check_file_md5sum "${AIRFLOW_SOURCES}/${FILE}"
@@ -257,7 +250,7 @@ function check_if_docker_build_is_needed() {
         export AIRFLOW_CONTAINER_DOCKER_BUILD_NEEDED="true"
     else
         set +e
-        for FILE in ${FILES_FOR_REBUILD_CHECK}
+        for FILE in "${FILES_FOR_REBUILD_CHECK[@]}"
         do
             if ! check_file_md5sum "${AIRFLOW_SOURCES}/${FILE}"; then
                 export AIRFLOW_CONTAINER_DOCKER_BUILD_NEEDED="true"
@@ -483,7 +476,7 @@ EOF
         fi
     else
         print_info
-        print_info "No need to rebuild - none of the important files changed: ${FILES_FOR_REBUILD_CHECK}"
+        print_info "No need to rebuild - none of the important files changed: ${FILES_FOR_REBUILD_CHECK[*]}"
         print_info
     fi
 }