LightGBM/.ci/test.sh

#!/bin/bash

set -e -E -o -u pipefail

# defaults
CONDA_ENV="test-env"
IN_UBUNTU_BASE_CONTAINER=${IN_UBUNTU_BASE_CONTAINER:-"false"}
METHOD=${METHOD:-""}
PRODUCES_ARTIFACTS=${PRODUCES_ARTIFACTS:-"false"}
SANITIZERS=${SANITIZERS:-""}

ARCH=$(uname -m)

LGB_VER=$(head -n 1 "${BUILD_DIRECTORY}/VERSION.txt")

if [[ $OS_NAME == "macos" ]] && [[ $COMPILER == "gcc" ]]; then
    export CXX=g++-12
    export CC=gcc-12
elif [[ $OS_NAME == "linux" ]] && [[ $COMPILER == "clang" ]]; then
    export CXX=clang++
    export CC=clang
elif [[ $OS_NAME == "linux" ]] && [[ $COMPILER == "clang-17" ]]; then
    export CXX=clang++-17
    export CC=clang-17
fi

if [[ $IN_UBUNTU_BASE_CONTAINER == "true" ]]; then
    export LANG="en_US.UTF-8"
    export LC_ALL="en_US.UTF-8"
fi

# Setting MACOSX_DEPLOYMENT_TARGET prevents CMake from building against too-new
# macOS features, and helps tools like Python build tools determine the appropriate
# wheel compatibility tags.
#
# ref:
#   * https://cmake.org/cmake/help/latest/envvar/MACOSX_DEPLOYMENT_TARGET.html
#   * https://github.com/scikit-build/scikit-build-core/blob/acb7d0346e4a05bcb47a4ea3939c705ab71e3145/src/scikit_build_core/builder/macos.py#L36
if [[ $ARCH == "x86_64" ]]; then
    export MACOSX_DEPLOYMENT_TARGET=10.15
else
    export MACOSX_DEPLOYMENT_TARGET=12.0
fi

if [[ "${TASK}" == "r-package" ]]; then
    bash "${BUILD_DIRECTORY}/.ci/test-r-package.sh" || exit 1
    exit 0
fi

if [[ "$TASK" == "cpp-tests" ]]; then
    cmake_args=(
        -DBUILD_CPP_TEST=ON
        -DUSE_DEBUG=ON
    )
    if [[ $METHOD == "with-sanitizers" ]]; then
        cmake_args+=("-DUSE_SANITIZER=ON")
        if [[ -n $SANITIZERS ]]; then
            cmake_args+=("-DENABLED_SANITIZERS=$SANITIZERS")
        fi
    fi
    cmake -B build -S . "${cmake_args[@]}"
    cmake --build build --target testlightgbm -j4 || exit 1
    ./testlightgbm || exit 1
    exit 0
fi

# including python=version[build=*cpython] to ensure that conda doesn't fall back to pypy
CONDA_PYTHON_REQUIREMENT="python=${PYTHON_VERSION}[build=*cpython]"

if [[ $TASK == "if-else" ]]; then
    conda create -q -y -n "${CONDA_ENV}" "${CONDA_PYTHON_REQUIREMENT}" numpy
    # shellcheck disable=SC1091
    source activate "${CONDA_ENV}"
    cmake -B build -S . || exit 1
    cmake --build build --target lightgbm -j4 || exit 1
    cd "$BUILD_DIRECTORY/tests/cpp_tests"
    ../../lightgbm config=train.conf convert_model_language=cpp convert_model=../../src/boosting/gbdt_prediction.cpp
    ../../lightgbm config=predict.conf output_result=origin.pred
    ../../lightgbm config=predict.conf output_result=ifelse.pred
    python test.py
    exit 0
fi

cd "${BUILD_DIRECTORY}"

if [[ $TASK == "swig" ]]; then
    cmake -B build -S . -DUSE_SWIG=ON
    cmake --build build -j4 || exit 1
    if [[ $OS_NAME == "linux" ]] && [[ $COMPILER == "gcc" ]]; then
        objdump -T ./lib_lightgbm.so > ./objdump.log || exit 1
        objdump -T ./lib_lightgbm_swig.so >> ./objdump.log || exit 1
        python ./.ci/check-dynamic-dependencies.py ./objdump.log || exit 1
    fi
    if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
        cp ./build/lightgbmlib.jar "${BUILD_ARTIFACTSTAGINGDIRECTORY}/lightgbmlib_${OS_NAME}.jar"
    fi
    exit 0
fi

if [[ $TASK == "lint" ]]; then
    pwsh -command "Install-Module -Name PSScriptAnalyzer -Scope CurrentUser -SkipPublisherCheck"
    echo "Linting PowerShell code"
    pwsh -file ./.ci/lint-powershell.ps1 || exit 1
    conda create -q -y -n "${CONDA_ENV}" \
        "${CONDA_PYTHON_REQUIREMENT}" \
        'biome>=1.9.3' \
        'cmakelint>=1.4.3' \
        'cpplint>=1.6.0' \
        'matplotlib-base>=3.9.1' \
        'mypy>=1.11.1' \
        'pre-commit>=3.8.0' \
        'pyarrow-core>=17.0' \
        'scikit-learn>=1.5.2' \
        'r-lintr>=3.1.2'
    # shellcheck disable=SC1091
    source activate "${CONDA_ENV}"
    echo "Linting Python and bash code"
    bash ./.ci/lint-python-bash.sh || exit 1
    echo "Linting R code"
    Rscript ./.ci/lint-r-code.R "${BUILD_DIRECTORY}" || exit 1
    echo "Linting C++ code"
    bash ./.ci/lint-cpp.sh || exit 1
    echo "Linting JavaScript code"
    bash ./.ci/lint-js.sh || exit 1
    exit 0
fi

if [[ $TASK == "check-docs" ]] || [[ $TASK == "check-links" ]]; then
    conda env create \
        -n "${CONDA_ENV}" \
        --file ./docs/env.yml || exit 1
    conda install \
        -q \
        -y \
        -n "${CONDA_ENV}" \
            'doxygen>=1.10.0' \
            'rstcheck>=6.2.4' || exit 1
    # shellcheck disable=SC1091
    source activate "${CONDA_ENV}"
    # check reStructuredText formatting
    find "${BUILD_DIRECTORY}/python-package" -type f -name "*.rst" \
        -exec rstcheck --report-level warning {} \+ || exit 1
    find "${BUILD_DIRECTORY}/docs" -type f -name "*.rst" \
        -exec rstcheck --report-level warning --ignore-directives=autoclass,autofunction,autosummary,doxygenfile {} \+ || exit 1
    # build docs
    make -C docs html || exit 1
    if [[ $TASK == "check-links" ]]; then
        # check docs for broken links
        pip install linkchecker
        linkchecker --config=.linkcheckerrc ./docs/_build/html/*.html || exit 1
        exit 0
    fi
    # check the consistency of parameters' descriptions and other stuff
    cp ./docs/Parameters.rst ./docs/Parameters-backup.rst
    cp ./src/io/config_auto.cpp ./src/io/config_auto-backup.cpp
    python ./.ci/parameter-generator.py || exit 1
    diff ./docs/Parameters-backup.rst ./docs/Parameters.rst || exit 1
    diff ./src/io/config_auto-backup.cpp ./src/io/config_auto.cpp || exit 1
    exit 0
fi

if [[ $PYTHON_VERSION == "3.7" ]]; then
    CONDA_REQUIREMENT_FILE="${BUILD_DIRECTORY}/.ci/conda-envs/ci-core-py37.txt"
elif [[ $PYTHON_VERSION == "3.8" ]]; then
    CONDA_REQUIREMENT_FILE="${BUILD_DIRECTORY}/.ci/conda-envs/ci-core-py38.txt"
else
    CONDA_REQUIREMENT_FILE="${BUILD_DIRECTORY}/.ci/conda-envs/ci-core.txt"
fi

conda create \
    -y \
    -n "${CONDA_ENV}" \
    --file "${CONDA_REQUIREMENT_FILE}" \
    "${CONDA_PYTHON_REQUIREMENT}" \
|| exit 1

# shellcheck disable=SC1091
source activate $CONDA_ENV

cd "${BUILD_DIRECTORY}"

if [[ $TASK == "sdist" ]]; then
    sh ./build-python.sh sdist || exit 1
    sh .ci/check-python-dists.sh ./dist || exit 1
    pip install "./dist/lightgbm-${LGB_VER}.tar.gz" -v || exit 1
    if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
        cp "./dist/lightgbm-${LGB_VER}.tar.gz" "${BUILD_ARTIFACTSTAGINGDIRECTORY}" || exit 1
    fi
    pytest ./tests/python_package_test || exit 1
    exit 0
elif [[ $TASK == "bdist" ]]; then
    if [[ $OS_NAME == "macos" ]]; then
        sh ./build-python.sh bdist_wheel || exit 1
        sh .ci/check-python-dists.sh ./dist || exit 1
        if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
            cp "$(echo "dist/lightgbm-${LGB_VER}-py3-none-macosx"*.whl)" "${BUILD_ARTIFACTSTAGINGDIRECTORY}" || exit 1
        fi
    else
        if [[ $ARCH == "x86_64" ]]; then
            PLATFORM="manylinux_2_28_x86_64"
        else
            PLATFORM="manylinux2014_$ARCH"
        fi
        sh ./build-python.sh bdist_wheel --integrated-opencl || exit 1
        # rename wheel, to fix scikit-build-core choosing the platform 'linux_aarch64' instead of
        # a manylinux tag
        mv \
            ./dist/*.whl \
            ./dist/tmp.whl || exit 1
        mv \
            ./dist/tmp.whl \
            "./dist/lightgbm-${LGB_VER}-py3-none-${PLATFORM}.whl" || exit 1
        sh .ci/check-python-dists.sh ./dist || exit 1
        if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
            cp "dist/lightgbm-${LGB_VER}-py3-none-${PLATFORM}.whl" "${BUILD_ARTIFACTSTAGINGDIRECTORY}" || exit 1
        fi
        # Make sure we can do both CPU and GPU; see tests/python_package_test/test_dual.py
        export LIGHTGBM_TEST_DUAL_CPU_GPU=1
    fi
    pip install -v ./dist/*.whl || exit 1
    pytest ./tests || exit 1
    exit 0
fi

if [[ $TASK == "gpu" ]]; then
    sed -i'.bak' 's/std::string device_type = "cpu";/std::string device_type = "gpu";/' ./include/LightGBM/config.h
    grep -q 'std::string device_type = "gpu"' ./include/LightGBM/config.h || exit 1  # make sure that changes were really done
    if [[ $METHOD == "pip" ]]; then
        sh ./build-python.sh sdist || exit 1
        sh .ci/check-python-dists.sh ./dist || exit 1
        pip install \
            -v \
            --config-settings=cmake.define.USE_GPU=ON \
            "./dist/lightgbm-${LGB_VER}.tar.gz" \
        || exit 1
        pytest ./tests/python_package_test || exit 1
        exit 0
    elif [[ $METHOD == "wheel" ]]; then
        sh ./build-python.sh bdist_wheel --gpu || exit 1
        sh ./.ci/check-python-dists.sh ./dist || exit 1
        pip install "$(echo "./dist/lightgbm-${LGB_VER}"*.whl)" -v || exit 1
        pytest ./tests || exit 1
        exit 0
    elif [[ $METHOD == "source" ]]; then
        cmake -B build -S . -DUSE_GPU=ON
    fi
elif [[ $TASK == "cuda" ]]; then
    sed -i'.bak' 's/std::string device_type = "cpu";/std::string device_type = "cuda";/' ./include/LightGBM/config.h
    grep -q 'std::string device_type = "cuda"' ./include/LightGBM/config.h || exit 1  # make sure that changes were really done
    # by default ``gpu_use_dp=false`` for efficiency. change to ``true`` here for exact results in ci tests
    sed -i'.bak' 's/gpu_use_dp = false;/gpu_use_dp = true;/' ./include/LightGBM/config.h
    grep -q 'gpu_use_dp = true' ./include/LightGBM/config.h || exit 1  # make sure that changes were really done
    if [[ $METHOD == "pip" ]]; then
        sh ./build-python.sh sdist || exit 1
        sh ./.ci/check-python-dists.sh ./dist || exit 1
        pip install \
            -v \
            --config-settings=cmake.define.USE_CUDA=ON \
            "./dist/lightgbm-${LGB_VER}.tar.gz" \
        || exit 1
        pytest ./tests/python_package_test || exit 1
        exit 0
    elif [[ $METHOD == "wheel" ]]; then
        sh ./build-python.sh bdist_wheel --cuda || exit 1
        sh ./.ci/check-python-dists.sh ./dist || exit 1
        pip install "$(echo "./dist/lightgbm-${LGB_VER}"*.whl)" -v || exit 1
        pytest ./tests || exit 1
        exit 0
    elif [[ $METHOD == "source" ]]; then
        cmake -B build -S . -DUSE_CUDA=ON
    fi
elif [[ $TASK == "mpi" ]]; then
    if [[ $METHOD == "pip" ]]; then
        sh ./build-python.sh sdist || exit 1
        sh ./.ci/check-python-dists.sh ./dist || exit 1
        pip install \
            -v \
            --config-settings=cmake.define.USE_MPI=ON \
            "./dist/lightgbm-${LGB_VER}.tar.gz" \
        || exit 1
        pytest ./tests/python_package_test || exit 1
        exit 0
    elif [[ $METHOD == "wheel" ]]; then
        sh ./build-python.sh bdist_wheel --mpi || exit 1
        sh ./.ci/check-python-dists.sh ./dist || exit 1
        pip install "$(echo "./dist/lightgbm-${LGB_VER}"*.whl)" -v || exit 1
        pytest ./tests || exit 1
        exit 0
    elif [[ $METHOD == "source" ]]; then
        cmake -B build -S . -DUSE_MPI=ON -DUSE_DEBUG=ON
    fi
else
    cmake -B build -S .
fi

cmake --build build --target _lightgbm -j4 || exit 1

sh ./build-python.sh install --precompile || exit 1
pytest ./tests || exit 1

if [[ $TASK == "regular" ]]; then
    if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
        if [[ $OS_NAME == "macos" ]]; then
            cp ./lib_lightgbm.dylib "${BUILD_ARTIFACTSTAGINGDIRECTORY}/lib_lightgbm.dylib"
        else
            if [[ $COMPILER == "gcc" ]]; then
                objdump -T ./lib_lightgbm.so > ./objdump.log || exit 1
                python ./.ci/check-dynamic-dependencies.py ./objdump.log || exit 1
            fi
            cp ./lib_lightgbm.so "${BUILD_ARTIFACTSTAGINGDIRECTORY}/lib_lightgbm.so"
        fi
    fi
    cd "$BUILD_DIRECTORY/examples/python-guide"
    sed -i'.bak' '/import lightgbm as lgb/a\
import matplotlib\
matplotlib.use\(\"Agg\"\)\
' plot_example.py  # prevent interactive window mode
    sed -i'.bak' 's/graph.render(view=True)/graph.render(view=False)/' plot_example.py
    # requirements for examples
    conda install -y -n $CONDA_ENV \
        'h5py>=3.10' \
        'ipywidgets>=8.1.2' \
        'notebook>=7.1.2'
    for f in *.py **/*.py; do python "${f}" || exit 1; done  # run all examples
    cd "$BUILD_DIRECTORY/examples/python-guide/notebooks"
    sed -i'.bak' 's/INTERACTIVE = False/assert False, \\"Interactive mode disabled\\"/' interactive_plot_example.ipynb
    jupyter nbconvert --ExecutePreprocessor.timeout=180 --to notebook --execute --inplace ./*.ipynb || exit 1  # run all notebooks

    # importing the library should succeed even if all optional dependencies are not present
    conda uninstall -n $CONDA_ENV --force --yes \
        cffi \
        dask \
        distributed \
        joblib \
        matplotlib-base \
        psutil \
        pyarrow \
        python-graphviz \
        scikit-learn || exit 1
    python -c "import lightgbm" || exit 1
fi