[AIRFLOW-6662] install dumb init (#7300)

* Revert "[AIRFLOW-6662] Switch to --init docker flag for signal propagation (#7278)"

This reverts commit d1bf343ffe.

* [AIRFLOW-6662] return back the dumb-init - installed by apt

We had stability problems with tests with --init flag so we are
going back to it

Dockerfile

@@ -57,6 +57,7 @@ RUN curl --fail --location https://deb.nodesource.com/setup_10.x | bash - \
            apt-utils \
            build-essential \
            dirmngr \
+           dumb-init \
            freetds-bin \
            freetds-dev \
            git \
@@ -353,6 +354,9 @@ RUN \
         && pip uninstall --yes apache-airflow; \
     fi
 
+# Link dumb-init for backwards compatibility (so that older images also work)
+RUN ln -sf /usr/bin/dumb-init /usr/local/bin/dumb-init
+
 # Install NPM dependencies here. The NPM dependencies don't change that often and we already have pip
 # installed dependencies in case of CI optimised build, so it is ok to install NPM deps here
 # Rather than after setup.py is added.
@@ -431,6 +435,6 @@ ENV PATH="${HOME}:${PATH}"
 
 EXPOSE 8080
 
-ENTRYPOINT ["/entrypoint.sh"]
+ENTRYPOINT ["/usr/bin/dumb-init", "--", "/entrypoint.sh"]
 
 CMD ["--help"]

scripts/ci/_utils.sh

@@ -640,30 +640,27 @@ function run_flake8() {
 
     if [[ "${#FILES[@]}" == "0" ]]; then
         docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \
-            --init \
+            --entrypoint "/usr/local/bin/dumb-init"  \
             --env PYTHONDONTWRITEBYTECODE \
             --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \
             --env AIRFLOW_CI_SILENT \
             --env HOST_USER_ID="$(id -ur)" \
             --env HOST_GROUP_ID="$(id -gr)" \
-            --entrypoint \
-            "/opt/airflow/scripts/ci/in_container/run_flake8.sh" \
             --rm \
             "${AIRFLOW_CI_IMAGE}" \
+            "--" "/opt/airflow/scripts/ci/in_container/run_flake8.sh" \
             | tee -a "${OUTPUT_LOG}"
     else
         docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \
-            --init \
+            --entrypoint "/usr/local/bin/dumb-init"  \
             --env PYTHONDONTWRITEBYTECODE \
             --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \
             --env AIRFLOW_CI_SILENT \
             --env HOST_USER_ID="$(id -ur)" \
             --env HOST_GROUP_ID="$(id -gr)" \
             --rm \
-            --entrypoint \
-            "/opt/airflow/scripts/ci/in_container/run_flake8.sh" \
             "${AIRFLOW_CI_IMAGE}" \
-            "${FILES[@]}" \
+            "--" "/opt/airflow/scripts/ci/in_container/run_flake8.sh" "${FILES[@]}" \
             | tee -a "${OUTPUT_LOG}"
     fi
 }
@@ -682,31 +679,29 @@ function run_bats_tests() {
 
 function run_docs() {
     docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" -t \
-            --init \
+            --entrypoint "/usr/local/bin/dumb-init"  \
             --env PYTHONDONTWRITEBYTECODE \
             --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \
             --env AIRFLOW_CI_SILENT \
             --env HOST_USER_ID="$(id -ur)" \
             --env HOST_GROUP_ID="$(id -gr)" \
-            --entrypoint \
-            "/opt/airflow/docs/build.sh" \
             --rm \
             "${AIRFLOW_CI_IMAGE}" \
+            "--" "/opt/airflow/docs/build.sh" \
             | tee -a "${OUTPUT_LOG}"
 }
 
 function run_check_license() {
     docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" -t \
-            --init \
+            --entrypoint "/usr/local/bin/dumb-init"  \
             --env PYTHONDONTWRITEBYTECODE \
             --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \
             --env AIRFLOW_CI_SILENT \
             --env HOST_USER_ID="$(id -ur)" \
             --env HOST_GROUP_ID="$(id -gr)" \
-            --entrypoint \
-            "/opt/airflow/scripts/ci/in_container/run_check_licence.sh" \
             --rm \
             "${AIRFLOW_CI_IMAGE}" \
+            "--" "/opt/airflow/scripts/ci/in_container/run_check_licence.sh" \
             | tee -a "${OUTPUT_LOG}"
 }
 
@@ -714,31 +709,27 @@ function run_mypy() {
     FILES=("$@")
     if [[ "${#FILES[@]}" == "0" ]]; then
         docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \
-            --init \
+            --entrypoint "/usr/local/bin/dumb-init"  \
             --env PYTHONDONTWRITEBYTECODE \
             --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \
             --env AIRFLOW_CI_SILENT \
             --env HOST_USER_ID="$(id -ur)" \
             --env HOST_GROUP_ID="$(id -gr)" \
             --rm \
-            --entrypoint \
-            "/opt/airflow/scripts/ci/in_container/run_mypy.sh" \
             "${AIRFLOW_CI_IMAGE}" \
-            "airflow" "tests" "docs" \
+            "--" "/opt/airflow/scripts/ci/in_container/run_mypy.sh" "airflow" "tests" "docs" \
             | tee -a "${OUTPUT_LOG}"
     else
         docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \
-            --init \
+            --entrypoint "/usr/local/bin/dumb-init" \
             --env PYTHONDONTWRITEBYTECODE \
             --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \
             --env AIRFLOW_CI_SILENT \
             --env HOST_USER_ID="$(id -ur)" \
             --env HOST_GROUP_ID="$(id -gr)" \
-            --entrypoint \
-            "/opt/airflow/scripts/ci/in_container/run_mypy.sh" \
             --rm \
             "${AIRFLOW_CI_IMAGE}" \
-            "${FILES[@]}" \
+            "--" "/opt/airflow/scripts/ci/in_container/run_mypy.sh" "${FILES[@]}" \
             | tee -a "${OUTPUT_LOG}"
     fi
 }
@@ -747,30 +738,27 @@ function run_pylint_main() {
     FILES=("$@")
     if [[ "${#FILES[@]}" == "0" ]]; then
         docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \
-            --init \
+            --entrypoint "/usr/local/bin/dumb-init"  \
             --env PYTHONDONTWRITEBYTECODE \
             --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \
             --env AIRFLOW_CI_SILENT \
             --env HOST_USER_ID="$(id -ur)" \
             --env HOST_GROUP_ID="$(id -gr)" \
             --rm \
-            --entrypoint \
-            "/opt/airflow/scripts/ci/in_container/run_pylint_main.sh" \
             "${AIRFLOW_CI_IMAGE}" \
+            "--" "/opt/airflow/scripts/ci/in_container/run_pylint_main.sh" \
             | tee -a "${OUTPUT_LOG}"
     else
         docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \
-            --init \
+            --entrypoint "/usr/local/bin/dumb-init" \
             --env PYTHONDONTWRITEBYTECODE \
             --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \
             --env AIRFLOW_CI_SILENT \
             --env HOST_USER_ID="$(id -ur)" \
             --env HOST_GROUP_ID="$(id -gr)" \
             --rm \
-            --entrypoint \
-            "/opt/airflow/scripts/ci/in_container/run_pylint_main.sh" \
             "${AIRFLOW_CI_IMAGE}" \
-            "${FILES[@]}" \
+            "--" "/opt/airflow/scripts/ci/in_container/run_pylint_main.sh" "${FILES[@]}" \
             | tee -a "${OUTPUT_LOG}"
     fi
 }
@@ -780,30 +768,27 @@ function run_pylint_tests() {
     FILES=("$@")
     if [[ "${#FILES[@]}" == "0" ]]; then
         docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \
-            --init \
+            --entrypoint "/usr/local/bin/dumb-init"  \
             --env PYTHONDONTWRITEBYTECODE \
             --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \
             --env AIRFLOW_CI_SILENT \
             --env HOST_USER_ID="$(id -ur)" \
             --env HOST_GROUP_ID="$(id -gr)" \
             --rm \
-            --entrypoint \
-            "/opt/airflow/scripts/ci/in_container/run_pylint_tests.sh" \
             "${AIRFLOW_CI_IMAGE}" \
+            "--" "/opt/airflow/scripts/ci/in_container/run_pylint_tests.sh" \
             | tee -a "${OUTPUT_LOG}"
     else
         docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \
-            --init \
+            --entrypoint "/usr/local/bin/dumb-init"  \
             --env PYTHONDONTWRITEBYTECODE \
             --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \
             --env AIRFLOW_CI_SILENT \
             --env HOST_USER_ID="$(id -ur)" \
             --env HOST_GROUP_ID="$(id -gr)" \
             --rm \
-            --entrypoint \
-            "/opt/airflow/scripts/ci/in_container/run_pylint_tests.sh" \
             "${AIRFLOW_CI_IMAGE}" \
-            "${FILES[@]}" \
+            "--" "/opt/airflow/scripts/ci/in_container/run_pylint_tests.sh" "${FILES[@]}" \
             | tee -a "${OUTPUT_LOG}"
     fi
 }
@@ -856,13 +841,15 @@ function filter_out_files_from_pylint_todo_list() {
 
 function refresh_pylint_todo() {
     docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \
-        --entrypoint /opt/airflow/scripts/ci/in_container/refresh_pylint_todo.sh \
         --env PYTHONDONTWRITEBYTECODE \
         --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \
         --env AIRFLOW_CI_SILENT \
         --env HOST_USER_ID="$(id -ur)" \
         --env HOST_GROUP_ID="$(id -gr)" \
-        "${AIRFLOW_CI_IMAGE}" | tee -a "${OUTPUT_LOG}"
+        --rm \
+        "${AIRFLOW_CI_IMAGE}" \
+        /opt/airflow/scripts/ci/in_container/refresh_pylint_todo.sh \
+        | tee -a "${OUTPUT_LOG}"
 }
 
 function rebuild_all_images_if_needed_and_confirmed() {

scripts/ci/docker-compose/backend-mysql.yml

@@ -25,7 +25,6 @@ services:
     depends_on:
       - mysql
   mysql:
-    init: true
     image: mysql:5.6
     environment:
       - MYSQL_ALLOW_EMPTY_PASSWORD=true

scripts/ci/docker-compose/backend-postgres.yml

@@ -25,7 +25,6 @@ services:
     depends_on:
       - postgres
   postgres:
-    init: true
     image: postgres:9.6
     environment:
       - POSTGRES_USER=postgres

scripts/ci/docker-compose/integration-cassandra.yml

@@ -18,7 +18,6 @@
 version: "2.2"
 services:
   cassandra:
-    init: true
     image: cassandra:3.0
     volumes:
       - /dev/urandom:/dev/random   # Required to get non-blocking entropy source

scripts/ci/docker-compose/integration-kerberos.yml

@@ -18,7 +18,6 @@
 version: "2.2"
 services:
   kerberos:
-    init: true
     image: godatadriven/krb5-kdc-server
     hostname: kerberos
     domainname: example.com

scripts/ci/docker-compose/integration-mongo.yml

@@ -18,7 +18,6 @@
 version: "2.2"
 services:
   mongo:
-    init: true
     image: mongo:3
     volumes:
       - /dev/urandom:/dev/random   # Required to get non-blocking entropy source

scripts/ci/docker-compose/integration-openldap.yml

@@ -18,7 +18,6 @@
 version: "2.2"
 services:
   openldap:
-    init: true
     image: osixia/openldap:1.2.0
     command: --copy-service
     environment:

scripts/ci/docker-compose/integration-rabbitmq.yml

@@ -18,7 +18,6 @@
 version: "2.2"
 services:
   rabbitmq:
-    init: true
     image: rabbitmq:3.7
     volumes:
       - /dev/urandom:/dev/random   # Required to get non-blocking entropy source

scripts/ci/docker-compose/integration-redis.yml

@@ -18,7 +18,6 @@
 version: "2.2"
 services:
   redis:
-    init: true
     image: redis:5.0.1
     volumes:
       - /dev/urandom:/dev/random   # Required to get non-blocking entropy source