Docker images are now consistently labelled and a bit smaller (#10387)

Extracted from #10368

.dockerignore

@@ -34,8 +34,20 @@
 !docs
 !licenses
 !metastore_browser
-!scripts
+
+# Add those folders to the context so that they are available in the CI container
+!scripts/ci/in_container
+!scripts/ci/pre_commit
+!scripts/prod
+!scripts/perf
+!scripts/tools
+
+# Add backport packages to the context
+!backport_packages
+
+# Add tests and kubernetes_tests to context.
 !tests
+!kubernetes_tests
 
 !.coveragerc
 !.rat-excludes

Dockerfile

@@ -54,12 +54,6 @@ ARG PYTHON_MAJOR_MINOR_VERSION="3.6"
 FROM ${PYTHON_BASE_IMAGE} as airflow-build-image
 SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"]
 
-LABEL org.apache.airflow.distro="debian"
-LABEL org.apache.airflow.distro.version="buster"
-LABEL org.apache.airflow.module="airflow"
-LABEL org.apache.airflow.component="airflow"
-LABEL org.apache.airflow.image="airflow-build-image"
-
 ARG PYTHON_BASE_IMAGE
 ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE}
 
@@ -171,6 +165,9 @@ ARG AIRFLOW_CONSTRAINTS_REFERENCE="constraints-master"
 ARG AIRFLOW_CONSTRAINTS_URL="https://raw.githubusercontent.com/apache/airflow/${AIRFLOW_CONSTRAINTS_REFERENCE}/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt"
 ENV AIRFLOW_CONSTRAINTS_URL=${AIRFLOW_CONSTRAINTS_URL}
 
+ENV PATH=${PATH}:/root/.local/bin
+RUN mkdir -p /root/.local/bin
+
 # In case of Production build image segment we want to pre-install master version of airflow
 # dependencies from github so that we do not have to always reinstall it from the scratch.
 RUN pip install --user \
@@ -202,8 +199,6 @@ ENV AIRFLOW_INSTALL_VERSION=${AIRFLOW_INSTALL_VERSION}
 
 WORKDIR /opt/airflow
 
-ENV PATH=${PATH}:/root/.local/bin
-
 RUN pip install --user "${AIRFLOW_INSTALL_SOURCES}[${AIRFLOW_EXTRAS}]${AIRFLOW_INSTALL_VERSION}" \
     --constraint "${AIRFLOW_CONSTRAINTS_URL}" && \
     if [ -n "${ADDITIONAL_PYTHON_DEPS}" ]; then pip install --user ${ADDITIONAL_PYTHON_DEPS} \
@@ -228,6 +223,20 @@ RUN AIRFLOW_SITE_PACKAGE="/root/.local/lib/python${PYTHON_MAJOR_MINOR_VERSION}/s
 RUN find /root/.local -executable -print0 | xargs --null chmod g+x && \
     find /root/.local -print0 | xargs --null chmod g+rw
 
+LABEL org.apache.airflow.distro="debian"
+LABEL org.apache.airflow.distro.version="buster"
+LABEL org.apache.airflow.module="airflow"
+LABEL org.apache.airflow.component="airflow"
+LABEL org.apache.airflow.image="airflow-build-image"
+
+ARG BUILD_ID
+ENV BUILD_ID=${BUILD_ID}
+ARG COMMIT_SHA
+ENV COMMIT_SHA=${COMMIT_SHA}
+
+LABEL org.apache.airflow.buildImage.buildId=${BUILD_ID}
+LABEL org.apache.airflow.buildImage.commitSha=${COMMIT_SHA}
+
 ##############################################################################################
 # This is the actual Airflow image - much smaller than the build one. We copy
 # installed Airflow and all it's dependencies from the build image to make it smaller.
@@ -372,5 +381,20 @@ EXPOSE 8080
 
 USER ${AIRFLOW_UID}
 
+ARG BUILD_ID
+ENV BUILD_ID=${BUILD_ID}
+ARG COMMIT_SHA
+ENV COMMIT_SHA=${COMMIT_SHA}
+
+LABEL org.apache.airflow.distro="debian"
+LABEL org.apache.airflow.distro.version="buster"
+LABEL org.apache.airflow.module="airflow"
+LABEL org.apache.airflow.component="airflow"
+LABEL org.apache.airflow.image="airflow"
+LABEL org.apache.airflow.uid="${AIRFLOW_UID}"
+LABEL org.apache.airflow.gid="${AIRFLOW_GID}"
+LABEL org.apache.airflow.mainImage.buildId=${BUILD_ID}
+LABEL org.apache.airflow.mainImage.commitSha=${COMMIT_SHA}
+
 ENTRYPOINT ["/usr/bin/dumb-init", "--", "/entrypoint"]
 CMD ["--help"]

Dockerfile.ci

@@ -281,20 +281,11 @@ RUN yarn --cwd airflow/www run prod
 COPY scripts/ci/in_container/entrypoint_ci.sh /entrypoint
 RUN chmod a+x /entrypoint
 
-# Copy selected subdirectories only
-COPY .github/ ${AIRFLOW_SOURCES}/.github/
-COPY metastore_browser/ ${AIRFLOW_SOURCES}/metastore_browser/
-COPY dags/ ${AIRFLOW_SOURCES}/dags/
-COPY common/ ${AIRFLOW_SOURCES}/common/
-COPY licenses/ ${AIRFLOW_SOURCES}/licenses/
-COPY scripts/ ${AIRFLOW_SOURCES}/scripts/
-COPY docs/ ${AIRFLOW_SOURCES}/docs/
-COPY tests/ ${AIRFLOW_SOURCES}/tests/
-COPY airflow/ ${AIRFLOW_SOURCES}/airflow/
-COPY .coveragerc .rat-excludes .flake8 pylintrc LICENSE MANIFEST.in NOTICE CHANGELOG.txt \
-     .github pytest.ini \
-     setup.cfg setup.py \
-     ${AIRFLOW_SOURCES}/
+# We can copy everything here. The Context is filtered by dockerignore. This makes sure we are not
+# copying over stuff that is accidentally generated or that we do not need (such as .egginfo)
+# if you want to add something that is missing and you expect to see it in the image you can
+# add it with ! in .dockerignore next to the airflow, test etc. directories there
+COPY . ${AIRFLOW_SOURCES}/
 
 # Install autocomplete for airflow
 RUN register-python-argcomplete airflow >> ~/.bashrc
@@ -318,6 +309,21 @@ ENV PATH="${HOME}:${PATH}"
 # Needed to stop Gunicorn from crashing when /tmp is now mounted from host
 ENV GUNICORN_CMD_ARGS="--worker-tmp-dir /dev/shm/"
 
+ARG BUILD_ID
+ENV BUILD_ID=${BUILD_ID}
+ARG COMMIT_SHA
+ENV COMMIT_SHA=${COMMIT_SHA}
+
+LABEL org.apache.airflow.distro="debian"
+LABEL org.apache.airflow.distro.version="buster"
+LABEL org.apache.airflow.module="airflow"
+LABEL org.apache.airflow.component="airflow"
+LABEL org.apache.airflow.image="airflow-ci"
+LABEL org.apache.airflow.uid="0"
+LABEL org.apache.airflow.gid="0"
+LABEL org.apache.airflow.buildId=${BUILD_ID}
+LABEL org.apache.airflow.commitSha=${COMMIT_SHA}
+
 EXPOSE 8080
 
 ENTRYPOINT ["/usr/bin/dumb-init", "--", "/entrypoint"]

scripts/ci/libraries/_build_images.sh

@@ -576,6 +576,8 @@ Docker building ${AIRFLOW_CI_IMAGE}.
         --build-arg ADDITIONAL_DEV_DEPS="${ADDITIONAL_DEV_DEPS}" \
         --build-arg ADDITIONAL_RUNTIME_DEPS="${ADDITIONAL_RUNTIME_DEPS}" \
         --build-arg UPGRADE_TO_LATEST_CONSTRAINTS="${UPGRADE_TO_LATEST_CONSTRAINTS}" \
+        --build-arg BUILD_ID="${CI_BUILD_ID}" \
+        --build-arg COMMIT_SHA="${COMMIT_SHA}" \
         "${DOCKER_CACHE_CI_DIRECTIVE[@]}" \
         -t "${AIRFLOW_CI_IMAGE}" \
         --target "main" \
@@ -724,6 +726,8 @@ function build_prod_image() {
         --build-arg ADDITIONAL_AIRFLOW_EXTRAS="${ADDITIONAL_AIRFLOW_EXTRAS}" \
         --build-arg ADDITIONAL_PYTHON_DEPS="${ADDITIONAL_PYTHON_DEPS}" \
         --build-arg ADDITIONAL_DEV_DEPS="${ADDITIONAL_DEV_DEPS}" \
+        --build-arg BUILD_ID="${CI_BUILD_ID}" \
+        --build-arg COMMIT_SHA="${COMMIT_SHA}" \
         "${DOCKER_CACHE_PROD_BUILD_DIRECTIVE[@]}" \
         -t "${AIRFLOW_PROD_BUILD_IMAGE}" \
         --target "airflow-build-image" \
@@ -740,6 +744,8 @@ function build_prod_image() {
         --build-arg AIRFLOW_BRANCH="${AIRFLOW_BRANCH_FOR_PYPI_PRELOADING}" \
         --build-arg AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS}" \
         --build-arg EMBEDDED_DAGS="${EMBEDDED_DAGS}" \
+        --build-arg BUILD_ID="${CI_BUILD_ID}" \
+        --build-arg COMMIT_SHA="${COMMIT_SHA}" \
         "${DOCKER_CACHE_PROD_DIRECTIVE[@]}" \
         -t "${AIRFLOW_PROD_IMAGE}" \
         --target "main" \

scripts/ci/libraries/_initialization.sh

@@ -220,6 +220,10 @@ function initialize_common_environment {
     # Namespace where airflow is installed via helm
     export HELM_AIRFLOW_NAMESPACE="airflow"
 
+    COMMIT_SHA="$(git rev-parse HEAD || echo "Unknown")"
+    export COMMIT_SHA
+
+    export CI_BUILD_ID="0"
 }
 
 # Retrieves CI environment variables needed - depending on the CI system we run it in.
@@ -233,7 +237,7 @@ function get_environment_for_builds_on_ci() {
     export CI_TARGET_BRANCH="master"
     export CI_SOURCE_REPO="apache/airflow"
     export CI_SOURCE_BRANCH="master"
-    export CI_BUILD_ID="default-build-id"
+    export CI_BUILD_ID="0"
     export CI_JOB_ID="default-job-id"
     if [[ ${CI:=} != "true" ]]; then
         echo