Make Production Dockerfile OpenShift-compatible (#9545)

OpenShift (and other Kubernetes platforms) often use the approach
that they start containers with random user and root group. This is
described in the https://docs.openshift.com/container-platform/3.7/creating_images/guidelines.html

All the files created by the "airflow" user are now belonging to
'root' group and the root group has the same access to those
files as the Airflow user.

Additionally, the random user gets automatically added
/etc/passwd entry which is name 'default'. The name of the user
can be set by setting the USER_NAME variable when starting the
container.

Closes #9248
Closes #8706

Dockerfile

@@ -214,8 +214,7 @@ RUN pip install --user "${AIRFLOW_INSTALL_SOURCES}[${AIRFLOW_EXTRAS}]${AIRFLOW_I
     find /root/.local/ -name '*.pyc' -print0 | xargs -0 rm -r && \
     find /root/.local/ -type d -name '__pycache__' -print0 | xargs -0 rm -r
 
-RUN \
-    AIRFLOW_SITE_PACKAGE="/root/.local/lib/python${PYTHON_MAJOR_MINOR_VERSION}/site-packages/airflow"; \
+RUN AIRFLOW_SITE_PACKAGE="/root/.local/lib/python${PYTHON_MAJOR_MINOR_VERSION}/site-packages/airflow"; \
     if [[ -f "${AIRFLOW_SITE_PACKAGE}/www_rbac/package.json" ]]; then \
         WWW_DIR="${AIRFLOW_SITE_PACKAGE}/www_rbac"; \
     elif [[ -f "${AIRFLOW_SITE_PACKAGE}/www/package.json" ]]; then \
@@ -227,6 +226,10 @@ RUN \
         rm -rf "${WWW_DIR}/node_modules"; \
     fi
 
+# make sure that all directories and files in .local are also group accessible
+RUN find /root/.local -executable -print0 | xargs --null chmod g+x && \
+    find /root/.local -print0 | xargs --null chmod g+rw
+
 ##############################################################################################
 # This is the actual Airflow image - much smaller than the build one. We copy
 # installed Airflow and all it's dependencies from the build image to make it smaller.
@@ -325,36 +328,47 @@ RUN pip install --upgrade pip==${PIP_VERSION}
 ENV AIRFLOW_UID=${AIRFLOW_UID}
 ENV AIRFLOW_GID=${AIRFLOW_GID}
 
+ENV AIRFLOW__CORE__LOAD_EXAMPLES="false"
+
+ARG AIRFLOW_USER_HOME_DIR=/home/airflow
+ENV AIRFLOW_USER_HOME_DIR=${AIRFLOW_USER_HOME_DIR}
+
 RUN addgroup --gid "${AIRFLOW_GID}" "airflow" && \
     adduser --quiet "airflow" --uid "${AIRFLOW_UID}" \
-        --ingroup "airflow" \
-        --home /home/airflow
+        --gid "${AIRFLOW_GID}" \
+        --home "${AIRFLOW_USER_HOME_DIR}"
 
 ARG AIRFLOW_HOME
 ENV AIRFLOW_HOME=${AIRFLOW_HOME}
 
+# Make Airflow files belong to the root group and are accessible. This is to accomodate the guidelines from
+# OpenShift https://docs.openshift.com/enterprise/3.0/creating_images/guidelines.html
 RUN mkdir -pv "${AIRFLOW_HOME}"; \
     mkdir -pv "${AIRFLOW_HOME}/dags"; \
     mkdir -pv "${AIRFLOW_HOME}/logs"; \
-    chown -R "airflow" "${AIRFLOW_HOME}"
+    chown -R "airflow:root" "${AIRFLOW_USER_HOME_DIR}" "${AIRFLOW_HOME}"; \
+    find "${AIRFLOW_HOME}" -executable -print0 | xargs --null chmod g+x && \
+        find "${AIRFLOW_HOME}" -print0 | xargs --null chmod g+rw
 
-COPY --chown=airflow:airflow --from=airflow-build-image /root/.local "/home/airflow/.local"
+COPY --chown=airflow:root --from=airflow-build-image /root/.local "${AIRFLOW_USER_HOME_DIR}/.local"
 
 COPY scripts/prod/entrypoint_prod.sh /entrypoint
 COPY scripts/prod/clean-logs.sh /clean-logs
 
 RUN chmod a+x /entrypoint /clean-logs
 
-USER airflow
+# Make /etc/passwd root-group-writeable so that user can be dynamically added by OpenShift
+# See https://github.com/apache/airflow/issues/9248
+RUN chmod g=u /etc/passwd
 
-ENV PATH="/home/airflow/.local/bin:${PATH}"
+ENV PATH="${AIRFLOW_USER_HOME_DIR}/.local/bin:${PATH}"
 ENV GUNICORN_CMD_ARGS="--worker-tmp-dir /dev/shm"
 
 WORKDIR ${AIRFLOW_HOME}
 
-ENV AIRFLOW__CORE__LOAD_EXAMPLES="false"
-
 EXPOSE 8080
 
+USER ${AIRFLOW_UID}
+
 ENTRYPOINT ["/usr/bin/dumb-init", "--", "/entrypoint"]
 CMD ["--help"]

IMAGES.rst

@@ -399,7 +399,12 @@ The following build arguments (``--build-arg`` in docker build command) can be u
 +------------------------------------------+------------------------------------------+------------------------------------------+
 | ``AIRFLOW_UID``                          | ``50000``                                | Airflow user UID                         |
 +------------------------------------------+------------------------------------------+------------------------------------------+
-| ``AIRFLOW_GID``                          | ``50000``                                | Airflow group GID                        |
+| ``AIRFLOW_GID``                          | ``50000``                                | Airflow group GID. Note that most files  |
+|                                          |                                          | created on behalf of airflow user belong |
+|                                          |                                          | to the ``root`` group (0) to keep        |
+|                                          |                                          | OpenShift Guidelines compatibility       |
++------------------------------------------+------------------------------------------+------------------------------------------+
+| ``AIRFLOW_USER_HOME_DIR``                | ``/home/airflow``                        | Home directory of the Airflow user       |
 +------------------------------------------+------------------------------------------+------------------------------------------+
 | ``PIP_VERSION``                          | ``19.0.2``                               | version of PIP to use                    |
 +------------------------------------------+------------------------------------------+------------------------------------------+
@@ -621,6 +626,11 @@ Using the PROD image
 
 The PROD image entrypoint works as follows:
 
+* In case the user is not "airflow" (with undefined user id) and the group id of the user is set to 0 (root),
+  then the user is dynamically added to /etc/passwd at entry using USER_NAME variable to define the user name.
+  This is in order to accommodate the
+  `OpenShift Guidelines<https://docs.openshift.com/enterprise/3.0/creating_images/guidelines.html>`_
+
 * If ``AIRFLOW__CORE__SQL_ALCHEMY_CONN`` variable is passed to the container and it is either mysql or postgres
   SQL alchemy connection, then the connection is checked and the script waits until the database is reachable.
 

scripts/ci/libraries/_build_images.sh

@@ -678,7 +678,6 @@ function build_prod_image() {
         --build-arg ADDITIONAL_AIRFLOW_EXTRAS="${ADDITIONAL_AIRFLOW_EXTRAS}" \
         --build-arg ADDITIONAL_PYTHON_DEPS="${ADDITIONAL_PYTHON_DEPS}" \
         --build-arg ADDITIONAL_DEV_DEPS="${ADDITIONAL_DEV_DEPS}" \
-        --build-arg ADDITIONAL_RUNTIME_DEPS="${ADDITIONAL_RUNTIME_DEPS}" \
         "${DOCKER_CACHE_PROD_BUILD_DIRECTIVE[@]}" \
         -t "${AIRFLOW_PROD_BUILD_IMAGE}" \
         --target "airflow-build-image" \

scripts/prod/entrypoint_prod.sh

@@ -90,6 +90,15 @@ function verify_db_connection {
     fi
 }
 
+if ! whoami &> /dev/null; then
+  if [[ -w /etc/passwd ]]; then
+    echo "${USER_NAME:-default}:x:$(id -u):0:${USER_NAME:-default} user:${AIRFLOW_USER_HOME_DIR}:/sbin/nologin" \
+        >> /etc/passwd
+  fi
+  export HOME="${AIRFLOW_USER_HOME_DIR}"
+fi
+
+
 # if no DB configured - use sqlite db by default
 AIRFLOW__CORE__SQL_ALCHEMY_CONN="${AIRFLOW__CORE__SQL_ALCHEMY_CONN:="sqlite:///${AIRFLOW_HOME}/airflow.db"}"