From 58d99dcee3aac2d2e06bfa2a9fb3f8b0165ff99d Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Tue, 19 Jan 2021 21:49:50 +0100 Subject: [PATCH] Remove chmod +x for installation script for docker build. (#13772) We've introduced chmod a+x for installation scripts in Dockerfiles. but this turned out to be a bad idea. This was to accomodate building on Azure Deveops which has filesystem that does not keep executable bit. But the side-effect of it that the layer of the script is invalidated when the permission is changed to +x on linux. The problem is that the script has locally (on checkout) different permissions depending on umask setting. Therefore changing permissions for the image to +a is not best. Instead we are running the scripts with bash directly, which does not require changing of executable bit. (cherry picked from commit 18d9320c2619dc569485a29fc71a62c26d29b3c9) --- Dockerfile | 19 ++++++------------- Dockerfile.ci | 20 ++++++-------------- 2 files changed, 12 insertions(+), 27 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5ec8dbbaab..c1d6de6131 100644 --- a/Dockerfile +++ b/Dockerfile @@ -146,8 +146,7 @@ ENV INSTALL_MYSQL_CLIENT=${INSTALL_MYSQL_CLIENT} COPY scripts/docker/install_mysql.sh /scripts/docker/install_mysql.sh COPY docker-context-files /docker-context-files # fix permission issue in Azure DevOps when running the script -RUN chmod a+x /scripts/docker/install_mysql.sh -RUN ./scripts/docker/install_mysql.sh dev +RUN bash ./scripts/docker/install_mysql.sh dev ARG AIRFLOW_REPO=apache/airflow ENV AIRFLOW_REPO=${AIRFLOW_REPO} @@ -194,8 +193,6 @@ ENV INSTALL_PROVIDERS_FROM_SOURCES=${INSTALL_PROVIDERS_FROM_SOURCES} # Only copy install_airflow_from_latest_master.sh to not invalidate cache on other script changes COPY scripts/docker/install_airflow_from_latest_master.sh /scripts/docker/install_airflow_from_latest_master.sh -# fix permission issue in Azure DevOps when running the script -RUN chmod a+x /scripts/docker/install_airflow_from_latest_master.sh # By default we do not upgrade to latest dependencies ARG UPGRADE_TO_NEWER_DEPENDENCIES="false" @@ -209,7 +206,7 @@ ENV UPGRADE_TO_NEWER_DEPENDENCIES=${UPGRADE_TO_NEWER_DEPENDENCIES} # account for removed dependencies (we do not install them in the first place) RUN if [[ ${AIRFLOW_PRE_CACHED_PIP_PACKAGES} == "true" && \ ${UPGRADE_TO_NEWER_DEPENDENCIES} == "false" ]]; then \ - /scripts/docker/install_airflow_from_latest_master.sh; \ + bash /scripts/docker/install_airflow_from_latest_master.sh; \ fi # By default we install latest airflow from PyPI so we do not need to copy sources of Airflow @@ -270,28 +267,24 @@ ARG CONTINUE_ON_PIP_CHECK_FAILURE="false" # Copy all install scripts here COPY scripts/docker/install*.sh /scripts/docker/ -# fix permission issue in Azure DevOps when running the script -RUN chmod a+x /scripts/docker/instal*.sh # hadolint ignore=SC2086, SC2010 RUN if [[ ${INSTALL_FROM_PYPI} == "true" ]]; then \ - /scripts/docker/install_airflow.sh; \ + bash /scripts/docker/install_airflow.sh; \ fi; \ if [[ ${INSTALL_FROM_DOCKER_CONTEXT_FILES} == "true" ]]; then \ - /scripts/docker/install_from_docker_context_files.sh; \ + bash /scripts/docker/install_from_docker_context_files.sh; \ fi; \ if [[ -n "${ADDITIONAL_PYTHON_DEPS}" ]]; then \ - /scripts/docker/install_additional_dependencies.sh; \ + bash /scripts/docker/install_additional_dependencies.sh; \ fi; \ find /root/.local/ -name '*.pyc' -print0 | xargs -0 rm -r || true ; \ find /root/.local/ -type d -name '__pycache__' -print0 | xargs -0 rm -r || true # Copy compile_www_assets.sh install scripts here COPY scripts/docker/compile_www_assets.sh /scripts/docker/compile_www_assets.sh -# fix permission issue in Azure DevOps when running the script -RUN chmod a+x /scripts/docker/compile_www_assets.sh -RUN /scripts/docker/compile_www_assets.sh +RUN bash /scripts/docker/compile_www_assets.sh # make sure that all directories and files in .local are also group accessible RUN find /root/.local -executable -print0 | xargs --null chmod g+x && \ diff --git a/Dockerfile.ci b/Dockerfile.ci index c0ad778942..b5ca211bd6 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -107,9 +107,7 @@ RUN mkdir -pv /usr/share/man/man1 \ # Only copy install_mysql.sh to not invalidate cache on other script changes COPY scripts/docker/install_mysql.sh /scripts/docker/install_mysql.sh -# fix permission issue in Azure DevOps when running the script -RUN chmod a+x /scripts/docker/install_mysql.sh -RUN /scripts/docker/install_mysql.sh dev +RUN bash /scripts/docker/install_mysql.sh dev RUN adduser airflow \ && echo "airflow ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/airflow \ @@ -216,7 +214,6 @@ ENV PATH "/opt/bats/bin/:${PATH}" # Additional scripts for managing BATS addons COPY scripts/docker/load.bash /opt/bats/lib/ -RUN chmod a+x /opt/bats/lib/load.bash # Optimizing installation of Cassandra driver # Speeds up building the image - cassandra driver without CYTHON saves around 10 minutes @@ -288,8 +285,6 @@ RUN pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}" # Only copy install_airflow_from_latest_master.sh to not invalidate cache on other script changes COPY scripts/docker/install_airflow_from_latest_master.sh /scripts/docker/install_airflow_from_latest_master.sh -# fix permission issue in Azure DevOps when running the script -RUN chmod a+x /scripts/docker/install_airflow_from_latest_master.sh ARG UPGRADE_TO_NEWER_DEPENDENCIES="false" ENV UPGRADE_TO_NEWER_DEPENDENCIES=${UPGRADE_TO_NEWER_DEPENDENCIES} @@ -303,7 +298,7 @@ ENV UPGRADE_TO_NEWER_DEPENDENCIES=${UPGRADE_TO_NEWER_DEPENDENCIES} # account for removed dependencies (we do not install them in the first place) RUN if [[ ${AIRFLOW_PRE_CACHED_PIP_PACKAGES} == "true" && \ ${UPGRADE_TO_NEWER_DEPENDENCIES} == "false" ]]; then \ - /scripts/docker/install_airflow_from_latest_master.sh; \ + bash /scripts/docker/install_airflow_from_latest_master.sh; \ fi # Generate random hex dump file so that we can determine whether it's faster to rebuild the image @@ -344,8 +339,6 @@ ARG CONTINUE_ON_PIP_CHECK_FAILURE="false" # Only copy install_airflow.sh to not invalidate cache on other script changes COPY scripts/docker/install_airflow.sh /scripts/docker/install_airflow.sh -# fix permission issue in Azure DevOps when running the script -RUN chmod a+x /scripts/docker/install_airflow.sh # The goal of this line is to install the dependencies from the most current setup.py from sources # This will be usually incremental small set of packages in CI optimized build, so it will be very fast @@ -354,20 +347,18 @@ RUN chmod a+x /scripts/docker/install_airflow.sh # But in cron job we will install latest versions matching setup.py to see if there is no breaking change # and push the constraints if everything is successful RUN if [[ ${INSTALL_FROM_PYPI} == "true" ]]; then \ - /scripts/docker/install_airflow.sh; \ + bash /scripts/docker/install_airflow.sh; \ fi # Only copy install_from_docker_context_files.sh to not invalidate cache on other script changes COPY scripts/docker/install_from_docker_context_files.sh /scripts/docker/install_from_docker_context_files.sh -# fix permission issue in Azure DevOps when running the script -RUN chmod a+x /scripts/docker/install_from_docker_context_files.sh # If wheel files are found in /docker-context-files during installation # they are also installed additionally to whatever is installed from Airflow. COPY docker-context-files/ /docker-context-files/ RUN if [[ ${INSTALL_FROM_DOCKER_CONTEXT_FILES} == "true" ]]; then \ - /scripts/docker/install_from_docker_context_files.sh; \ + bash /scripts/docker/install_from_docker_context_files.sh; \ fi # Copy all the www/ files we need to compile assets. Done as two separate COPY @@ -375,8 +366,9 @@ RUN if [[ ${INSTALL_FROM_DOCKER_CONTEXT_FILES} == "true" ]]; then \ COPY airflow/www/webpack.config.js ${AIRFLOW_SOURCES}/airflow/www/ COPY airflow/www/static ${AIRFLOW_SOURCES}/airflow/www/static/ COPY airflow/www/compile_assets.sh ${AIRFLOW_SOURCES}/airflow/www/compile_assets.sh + # Package JS/css for production -RUN airflow/www/compile_assets.sh +RUN bash airflow/www/compile_assets.sh COPY scripts/in_container/entrypoint_ci.sh /entrypoint RUN chmod a+x /entrypoint