Build normal/backport providers based on correct list (#12848)

This was using the backport list when trying to build the normal list,
leading to papermill package not getting built.

I have re-introduced the exit removed in #12841, and also done a little
bit of drive-by-tidying

scripts/in_container/run_install_and_test_provider_packages.sh

@@ -79,18 +79,24 @@ function discover_all_provider_packages() {
     echo Listing available providers via 'airflow providers list'
     echo
 
-    airflow providers list
+    # Columns is to force it wider, so it doesn't wrap at 80 characters
+    COLUMNS=180 airflow providers list
 
-    local expected_number_of_providers=60
+    local expected_number_of_providers=61
     local actual_number_of_providers
-    actual_number_of_providers=$(airflow providers list --output table | grep -c apache-airflow-providers | xargs)
+    actual_providers=$(airflow providers list --output yaml | grep package_name)
-    if [[ ${actual_number_of_providers} != "${expected_number_of_providers}" ]]; then
+    actual_number_of_providers=$(wc -l <<<"$actual_providers")
+    if [[ "${actual_number_of_providers}" != "${expected_number_of_providers}" ]]; then
         echo
-        echo  "${COLOR_RED_ERROR} Number of providers installed is wrong  ${COLOR_RESET}"
+        echo  "${COLOR_RED_ERROR}Number of providers installed is wrong${COLOR_RESET}"
         echo "Expected number was '${expected_number_of_providers}' and got '${actual_number_of_providers}'"
         echo
         echo "Either increase the number of providers if you added one or diagnose and fix the problem."
         echo
+        echo "Providers were:"
+        echo
+        echo "$actual_providers"
+        exit 1
     fi
 }
 
@@ -111,6 +117,7 @@ function discover_all_hooks() {
         echo
         echo "Either increase the number of hooks if you added one or diagnose and fix the problem."
         echo
+        exit 1
     fi
 }
 
@@ -131,6 +138,7 @@ function discover_all_extra_links() {
         echo
         echo "Either increase the number of links if you added one or diagnose and fix the problem."
         echo
+        exit 1
     fi
 }
 

scripts/in_container/run_prepare_provider_packages.sh

@@ -20,12 +20,7 @@
 
 setup_provider_packages
 
-LIST_OF_DIRS_FILE=$(mktemp)
 
-cd "${AIRFLOW_SOURCES}/airflow/providers" || exit 1
-
-find . -type d | sed 's/.\///; s/\//\./g' | grep -E 'hooks|operators|sensors|secrets|utils' \
-    > "${LIST_OF_DIRS_FILE}"
 
 cd "${AIRFLOW_SOURCES}/provider_packages" || exit 1
 
@@ -34,12 +29,25 @@ REFACTOR_PROVIDER_PACKAGES_PY="${AIRFLOW_SOURCES}/dev/provider_packages/refactor
 
 verify_suffix_versions_for_package_preparation
 
-if [[ -z "$*" ]]; then
+if [[ ${BACKPORT_PACKAGES} == "true" ]]; then
-    PROVIDERS_PACKAGES=$(python3 "${PREPARE_PROVIDER_PACKAGES_PY}" list-providers-packages)
+  list_subcmd="list-backportable-packages"
+else
+  list_subcmd="list-providers-packages"
+fi
 
+function check_missing_providers() {
     PACKAGE_ERROR="false"
+
+    pushd "${AIRFLOW_SOURCES}/airflow/providers" >/dev/null || exit 1
+
+    LIST_OF_DIRS_FILE=$(mktemp)
+    find . -type d | sed 's!./!!; s!/!.!g' | grep -E 'hooks|operators|sensors|secrets|utils' \
+        > "${LIST_OF_DIRS_FILE}"
+
+    popd >/dev/null
+
     # Check if all providers are included
-    for PACKAGE in ${PROVIDERS_PACKAGES}
+    for PACKAGE in "${PROVIDER_PACKAGES[@]}"
     do
         if ! grep -E "^${PACKAGE}" <"${LIST_OF_DIRS_FILE}" >/dev/null; then
             echo "The package ${PACKAGE} is not available in providers dir"
@@ -54,16 +62,30 @@ if [[ -z "$*" ]]; then
         exit 1
     fi
 
-    NUM_LINES=$(wc -l "${LIST_OF_DIRS_FILE}" | awk '{ print $1 }')
+    if [[ $(wc -l < "${LIST_OF_DIRS_FILE}") != "0" ]]; then
-    if [[ ${NUM_LINES} != "0" ]]; then
         echo "ERROR! Some folders from providers package are not defined"
         echo "       Please add them to dev/provider_packages/prepare_provider_packages.py:"
         echo
         cat "${LIST_OF_DIRS_FILE}"
         echo
+
+        rm "$LIST_OF_DIRS_FILE"
         exit 1
     fi
-    PROVIDER_PACKAGES=$(python3 "${PREPARE_PROVIDER_PACKAGES_PY}" list-backportable-packages)
+    rm "$LIST_OF_DIRS_FILE"
+}
+
+if [[ -z "$*" ]]; then
+    PROVIDER_PACKAGES=()
+    while IFS='' read -r line; do PROVIDER_PACKAGES+=("$line"); done < <(
+      python3 "${PREPARE_PROVIDER_PACKAGES_PY}" "$list_subcmd"
+    )
+
+    if [[ "$BACKPORT_PACKAGES" != "true" ]]; then
+        # Don't check for missing packages when we are building backports -- we have filtered someout, and the
+        # non-backport build will check for any missing.
+        check_missing_providers
+    fi
 else
     if [[ "$1" == "--help" ]]; then
         echo
@@ -76,7 +98,7 @@ else
         echo
         exit
     fi
-    PROVIDER_PACKAGES="$*"
+    PROVIDER_PACKAGES=("$@")
 fi
 
 if [[ ${BACKPORT_PACKAGES} == "true" ]]; then
@@ -93,10 +115,9 @@ python3 "${REFACTOR_PROVIDER_PACKAGES_PY}"
 
 rm -rf dist/*
 
-for PROVIDER_PACKAGE in ${PROVIDER_PACKAGES}
+for PROVIDER_PACKAGE in "${PROVIDER_PACKAGES[@]}"
 do
-    rm -rf -- *.egg-info
+    rm -rf -- *.egg-info build/
-    rm -rf build/
     LOG_FILE=$(mktemp)
     python3 "${PREPARE_PROVIDER_PACKAGES_PY}" generate-setup-files "${PROVIDER_PACKAGE}"
     echo "==================================================================================="