incubator-airflow/docs/build.sh

#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

set -euo pipefail

MY_DIR="$(cd "$(dirname "$0")" && pwd)"
pushd "${MY_DIR}" &>/dev/null || exit 1

echo
echo "Working in ${MY_DIR} folder"
echo


if [[ -f /.dockerenv ]]; then
    # This script can be run both - in container and outside of it.
    # Here we are inside the container which means that we should (when the host is Linux)
    # fix permissions of the _build and _api folders via sudo.
    # Those files are mounted from the host via docs folder and we might not have permissions to
    # write to those directories (and remove the _api folder).
    # We know we have sudo capabilities inside the container.
    echo "Creating the _build and _api folders in case they do not exist"
    sudo mkdir -pv _build
    sudo mkdir -pv _api
    echo "Created the _build and _api folders in case they do not exist"
else
    # We are outside the container so we simply make sure that the directories exist
    echo "Creating the _build and _api folders in case they do not exist"
    mkdir -pv _build
    mkdir -pv _api
    echo "Creating the _build and _api folders in case they do not exist"
fi

echo "Removing content of the _build and _api folders"
rm -rf _build/*
rm -rf _api/*
echo "Removed content of the _build and _api folders"


set +e
# shellcheck disable=SC2063
NUM_INCORRECT_USE_LITERALINCLUDE=$(grep -inR --include \*.rst 'literalinclude::.\+example_dags' . | \
    tee /dev/tty |
    wc -l |\
    tr -d '[:space:]')
set -e

echo
echo "Checking for presence of literalinclude in example DAGs"
echo

if [[ "${NUM_INCORRECT_USE_LITERALINCLUDE}" -ne "0" ]]; then
    echo
    echo "Unexpected problems found in the documentation. "
    echo "You should use a exampleinclude directive to include example DAGs."
    echo "Currently, ${NUM_INCORRECT_USE_LITERALINCLUDE} problem found."
    echo
    exit 1
else
    echo
    echo "No literalincludes in example DAGs found"
    echo
fi

echo
echo "Checking the status of the operators-and-hooks-ref.rst file."
echo

mapfile -t DEPRECATED_MODULES < <(grep -R -i -l 'This module is deprecated.' ../airflow --include '*.py' | \
    cut -d "/" -f 2- | \
    sort | \
    uniq | \
    cut -d "." -f 1 | \
    sed "s#/#.#g")

IGNORED_MISSING_MODULES=('airflow.gcp.hooks.base')

mapfile -t ALL_MODULES < <(find ../airflow/{,gcp/,contrib/,providers/*/*/}{operators,sensors,hooks} -name "*.py" | \
    grep -v "__init__" | \
    grep -v "__pycache__" | \
    cut -d "/" -f 2- | \
    cut -d "." -f 1 | \
    sed "s#/#.#g" | \
    sort | \
    uniq | \
    grep -vf <(printf '%s\n' "${DEPRECATED_MODULES[@]}") |\
    grep -vf <(printf '%s\n' "${IGNORED_MISSING_MODULES[@]}"))

# shellcheck disable=SC2002
mapfile -t CURRENT_MODULES < <(cat operators-and-hooks-ref.rst | \
    grep ":mod:" | \
    cut -d '`' -f 2 | \
    sort | \
    uniq | \
    grep -v "__pycache__")

mapfile -t MISSING_MODULES < \
    <(\
        comm -2 -3 \
        <(printf '%s\n' "${ALL_MODULES[@]}" | sort ) \
        <(printf '%s\n' "${CURRENT_MODULES[@]}" | sort)
    )

if [[ "${#MISSING_MODULES[@]}" -ne "0" ]]; then
    echo
    echo "Unexpected problems found in the documentation."
    echo "You should try to keep the list of operators and hooks up to date."
    echo
    echo "Missing modules:"
    printf '%s\n' "${MISSING_MODULES[@]}"
    echo
    echo "Please add this module to operators-and-hooks-ref.rst file."
    echo
    exit 1
else
    echo
    echo "The operators-and-hooks-ref.rst file seems to be in good condition."
    echo
fi


SUCCEED_LINE=$(make html |\
    tee /dev/tty |\
    grep 'build succeeded' |\
    head -1)

NUM_CURRENT_WARNINGS=$(echo "${SUCCEED_LINE}" |\
    sed -E 's/build succeeded, ([0-9]+) warnings?\./\1/g')

if [[  -f /.dockerenv ]]; then
    # We are inside the container which means that we should fix back the permissions of the
    # _build and _api folder files, so that they can be accessed by the host user
    # The _api folder should be deleted by then but just in case we should change the ownership
    echo "Changing ownership of docs/_build folder back to ${HOST_USER_ID}:${HOST_GROUP_ID}"
    sudo chown "${HOST_USER_ID}":"${HOST_GROUP_ID}" _build
    if [[ -d _api ]]; then
        sudo chown "${HOST_USER_ID}":"${HOST_GROUP_ID}" _api
    fi
    echo "Changed ownership of docs/_build folder back to ${HOST_USER_ID}:${HOST_GROUP_ID}"
fi


if echo "${SUCCEED_LINE}" | grep -q "warning"; then
    echo
    echo "Unexpected problems found in the documentation. "
    echo "Currently, ${NUM_CURRENT_WARNINGS} warnings found. "
    echo
    exit 1
fi

popd &>/dev/null || exit 1