diff --git a/.dockerignore b/.dockerignore index 30aae39b4e..d9e4b43562 100644 --- a/.dockerignore +++ b/.dockerignore @@ -34,8 +34,20 @@ !docs !licenses !metastore_browser -!scripts + +# Add those folders to the context so that they are available in the CI container +!scripts/ci/in_container +!scripts/ci/pre_commit +!scripts/prod +!scripts/perf +!scripts/tools + +# Add backport packages to the context +!backport_packages + +# Add tests and kubernetes_tests to context. !tests +!kubernetes_tests !.coveragerc !.rat-excludes diff --git a/Dockerfile b/Dockerfile index a50aa745e0..76a42764af 100644 --- a/Dockerfile +++ b/Dockerfile @@ -54,12 +54,6 @@ ARG PYTHON_MAJOR_MINOR_VERSION="3.6" FROM ${PYTHON_BASE_IMAGE} as airflow-build-image SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"] -LABEL org.apache.airflow.distro="debian" -LABEL org.apache.airflow.distro.version="buster" -LABEL org.apache.airflow.module="airflow" -LABEL org.apache.airflow.component="airflow" -LABEL org.apache.airflow.image="airflow-build-image" - ARG PYTHON_BASE_IMAGE ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE} @@ -171,6 +165,9 @@ ARG AIRFLOW_CONSTRAINTS_REFERENCE="constraints-master" ARG AIRFLOW_CONSTRAINTS_URL="https://raw.githubusercontent.com/apache/airflow/${AIRFLOW_CONSTRAINTS_REFERENCE}/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt" ENV AIRFLOW_CONSTRAINTS_URL=${AIRFLOW_CONSTRAINTS_URL} +ENV PATH=${PATH}:/root/.local/bin +RUN mkdir -p /root/.local/bin + # In case of Production build image segment we want to pre-install master version of airflow # dependencies from github so that we do not have to always reinstall it from the scratch. RUN pip install --user \ @@ -202,8 +199,6 @@ ENV AIRFLOW_INSTALL_VERSION=${AIRFLOW_INSTALL_VERSION} WORKDIR /opt/airflow -ENV PATH=${PATH}:/root/.local/bin - RUN pip install --user "${AIRFLOW_INSTALL_SOURCES}[${AIRFLOW_EXTRAS}]${AIRFLOW_INSTALL_VERSION}" \ --constraint "${AIRFLOW_CONSTRAINTS_URL}" && \ if [ -n "${ADDITIONAL_PYTHON_DEPS}" ]; then pip install --user ${ADDITIONAL_PYTHON_DEPS} \ @@ -228,6 +223,20 @@ RUN AIRFLOW_SITE_PACKAGE="/root/.local/lib/python${PYTHON_MAJOR_MINOR_VERSION}/s RUN find /root/.local -executable -print0 | xargs --null chmod g+x && \ find /root/.local -print0 | xargs --null chmod g+rw +LABEL org.apache.airflow.distro="debian" +LABEL org.apache.airflow.distro.version="buster" +LABEL org.apache.airflow.module="airflow" +LABEL org.apache.airflow.component="airflow" +LABEL org.apache.airflow.image="airflow-build-image" + +ARG BUILD_ID +ENV BUILD_ID=${BUILD_ID} +ARG COMMIT_SHA +ENV COMMIT_SHA=${COMMIT_SHA} + +LABEL org.apache.airflow.buildImage.buildId=${BUILD_ID} +LABEL org.apache.airflow.buildImage.commitSha=${COMMIT_SHA} + ############################################################################################## # This is the actual Airflow image - much smaller than the build one. We copy # installed Airflow and all it's dependencies from the build image to make it smaller. @@ -372,5 +381,20 @@ EXPOSE 8080 USER ${AIRFLOW_UID} +ARG BUILD_ID +ENV BUILD_ID=${BUILD_ID} +ARG COMMIT_SHA +ENV COMMIT_SHA=${COMMIT_SHA} + +LABEL org.apache.airflow.distro="debian" +LABEL org.apache.airflow.distro.version="buster" +LABEL org.apache.airflow.module="airflow" +LABEL org.apache.airflow.component="airflow" +LABEL org.apache.airflow.image="airflow" +LABEL org.apache.airflow.uid="${AIRFLOW_UID}" +LABEL org.apache.airflow.gid="${AIRFLOW_GID}" +LABEL org.apache.airflow.mainImage.buildId=${BUILD_ID} +LABEL org.apache.airflow.mainImage.commitSha=${COMMIT_SHA} + ENTRYPOINT ["/usr/bin/dumb-init", "--", "/entrypoint"] CMD ["--help"] diff --git a/Dockerfile.ci b/Dockerfile.ci index 608709f193..dbb1ce876d 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -281,20 +281,11 @@ RUN yarn --cwd airflow/www run prod COPY scripts/ci/in_container/entrypoint_ci.sh /entrypoint RUN chmod a+x /entrypoint -# Copy selected subdirectories only -COPY .github/ ${AIRFLOW_SOURCES}/.github/ -COPY metastore_browser/ ${AIRFLOW_SOURCES}/metastore_browser/ -COPY dags/ ${AIRFLOW_SOURCES}/dags/ -COPY common/ ${AIRFLOW_SOURCES}/common/ -COPY licenses/ ${AIRFLOW_SOURCES}/licenses/ -COPY scripts/ ${AIRFLOW_SOURCES}/scripts/ -COPY docs/ ${AIRFLOW_SOURCES}/docs/ -COPY tests/ ${AIRFLOW_SOURCES}/tests/ -COPY airflow/ ${AIRFLOW_SOURCES}/airflow/ -COPY .coveragerc .rat-excludes .flake8 pylintrc LICENSE MANIFEST.in NOTICE CHANGELOG.txt \ - .github pytest.ini \ - setup.cfg setup.py \ - ${AIRFLOW_SOURCES}/ +# We can copy everything here. The Context is filtered by dockerignore. This makes sure we are not +# copying over stuff that is accidentally generated or that we do not need (such as .egginfo) +# if you want to add something that is missing and you expect to see it in the image you can +# add it with ! in .dockerignore next to the airflow, test etc. directories there +COPY . ${AIRFLOW_SOURCES}/ # Install autocomplete for airflow RUN register-python-argcomplete airflow >> ~/.bashrc @@ -318,6 +309,21 @@ ENV PATH="${HOME}:${PATH}" # Needed to stop Gunicorn from crashing when /tmp is now mounted from host ENV GUNICORN_CMD_ARGS="--worker-tmp-dir /dev/shm/" +ARG BUILD_ID +ENV BUILD_ID=${BUILD_ID} +ARG COMMIT_SHA +ENV COMMIT_SHA=${COMMIT_SHA} + +LABEL org.apache.airflow.distro="debian" +LABEL org.apache.airflow.distro.version="buster" +LABEL org.apache.airflow.module="airflow" +LABEL org.apache.airflow.component="airflow" +LABEL org.apache.airflow.image="airflow-ci" +LABEL org.apache.airflow.uid="0" +LABEL org.apache.airflow.gid="0" +LABEL org.apache.airflow.buildId=${BUILD_ID} +LABEL org.apache.airflow.commitSha=${COMMIT_SHA} + EXPOSE 8080 ENTRYPOINT ["/usr/bin/dumb-init", "--", "/entrypoint"] diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh index 715b652726..5469b21418 100644 --- a/scripts/ci/libraries/_build_images.sh +++ b/scripts/ci/libraries/_build_images.sh @@ -576,6 +576,8 @@ Docker building ${AIRFLOW_CI_IMAGE}. --build-arg ADDITIONAL_DEV_DEPS="${ADDITIONAL_DEV_DEPS}" \ --build-arg ADDITIONAL_RUNTIME_DEPS="${ADDITIONAL_RUNTIME_DEPS}" \ --build-arg UPGRADE_TO_LATEST_CONSTRAINTS="${UPGRADE_TO_LATEST_CONSTRAINTS}" \ + --build-arg BUILD_ID="${CI_BUILD_ID}" \ + --build-arg COMMIT_SHA="${COMMIT_SHA}" \ "${DOCKER_CACHE_CI_DIRECTIVE[@]}" \ -t "${AIRFLOW_CI_IMAGE}" \ --target "main" \ @@ -724,6 +726,8 @@ function build_prod_image() { --build-arg ADDITIONAL_AIRFLOW_EXTRAS="${ADDITIONAL_AIRFLOW_EXTRAS}" \ --build-arg ADDITIONAL_PYTHON_DEPS="${ADDITIONAL_PYTHON_DEPS}" \ --build-arg ADDITIONAL_DEV_DEPS="${ADDITIONAL_DEV_DEPS}" \ + --build-arg BUILD_ID="${CI_BUILD_ID}" \ + --build-arg COMMIT_SHA="${COMMIT_SHA}" \ "${DOCKER_CACHE_PROD_BUILD_DIRECTIVE[@]}" \ -t "${AIRFLOW_PROD_BUILD_IMAGE}" \ --target "airflow-build-image" \ @@ -740,6 +744,8 @@ function build_prod_image() { --build-arg AIRFLOW_BRANCH="${AIRFLOW_BRANCH_FOR_PYPI_PRELOADING}" \ --build-arg AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS}" \ --build-arg EMBEDDED_DAGS="${EMBEDDED_DAGS}" \ + --build-arg BUILD_ID="${CI_BUILD_ID}" \ + --build-arg COMMIT_SHA="${COMMIT_SHA}" \ "${DOCKER_CACHE_PROD_DIRECTIVE[@]}" \ -t "${AIRFLOW_PROD_IMAGE}" \ --target "main" \ diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh index d3fa761c42..f4c7876365 100644 --- a/scripts/ci/libraries/_initialization.sh +++ b/scripts/ci/libraries/_initialization.sh @@ -220,6 +220,10 @@ function initialize_common_environment { # Namespace where airflow is installed via helm export HELM_AIRFLOW_NAMESPACE="airflow" + COMMIT_SHA="$(git rev-parse HEAD || echo "Unknown")" + export COMMIT_SHA + + export CI_BUILD_ID="0" } # Retrieves CI environment variables needed - depending on the CI system we run it in. @@ -233,7 +237,7 @@ function get_environment_for_builds_on_ci() { export CI_TARGET_BRANCH="master" export CI_SOURCE_REPO="apache/airflow" export CI_SOURCE_BRANCH="master" - export CI_BUILD_ID="default-build-id" + export CI_BUILD_ID="0" export CI_JOB_ID="default-job-id" if [[ ${CI:=} != "true" ]]; then echo