[ci] reduce repetition of BUILD_DIRECTORY in CI scripts (#6497)

This commit is contained in:
James Lamb 2024-06-21 12:16:41 -05:00 коммит произвёл GitHub
Родитель fd8f799449
Коммит bf2641d8cb
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: B5690EEEBB952194
3 изменённых файлов: 88 добавлений и 86 удалений

Просмотреть файл

@ -11,7 +11,7 @@ SANITIZERS=${SANITIZERS:-""}
ARCH=$(uname -m)
LGB_VER=$(head -n 1 ${BUILD_DIRECTORY}/VERSION.txt)
LGB_VER=$(head -n 1 "${BUILD_DIRECTORY}/VERSION.txt")
if [[ $OS_NAME == "macos" ]] && [[ $COMPILER == "gcc" ]]; then
export CXX=g++-11
@ -43,7 +43,7 @@ else
fi
if [[ "${TASK}" == "r-package" ]] || [[ "${TASK}" == "r-rchk" ]]; then
bash ${BUILD_DIRECTORY}/.ci/test_r_package.sh || exit 1
bash "${BUILD_DIRECTORY}/.ci/test_r_package.sh" || exit 1
exit 0
fi
@ -70,27 +70,31 @@ if [[ $TASK == "if-else" ]]; then
source activate $CONDA_ENV
cmake -B build -S . || exit 1
cmake --build build --target lightgbm -j4 || exit 1
cd $BUILD_DIRECTORY/tests/cpp_tests && ../../lightgbm config=train.conf convert_model_language=cpp convert_model=../../src/boosting/gbdt_prediction.cpp && ../../lightgbm config=predict.conf output_result=origin.pred || exit 1
cd $BUILD_DIRECTORY/tests/cpp_tests && ../../lightgbm config=predict.conf output_result=ifelse.pred && python test.py || exit 1
cd "$BUILD_DIRECTORY/tests/cpp_tests"
../../lightgbm config=train.conf convert_model_language=cpp convert_model=../../src/boosting/gbdt_prediction.cpp
../../lightgbm config=predict.conf output_result=origin.pred
../../lightgbm config=predict.conf output_result=ifelse.pred
python test.py
exit 0
fi
cd "${BUILD_DIRECTORY}"
if [[ $TASK == "swig" ]]; then
cmake -B build -S . -DUSE_SWIG=ON
cmake --build build -j4 || exit 1
if [[ $OS_NAME == "linux" ]] && [[ $COMPILER == "gcc" ]]; then
objdump -T $BUILD_DIRECTORY/lib_lightgbm.so > $BUILD_DIRECTORY/objdump.log || exit 1
objdump -T $BUILD_DIRECTORY/lib_lightgbm_swig.so >> $BUILD_DIRECTORY/objdump.log || exit 1
python $BUILD_DIRECTORY/helpers/check_dynamic_dependencies.py $BUILD_DIRECTORY/objdump.log || exit 1
objdump -T ./lib_lightgbm.so > ./objdump.log || exit 1
objdump -T ./lib_lightgbm_swig.so >> ./objdump.log || exit 1
python ./helpers/check_dynamic_dependencies.py ./objdump.log || exit 1
fi
if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
cp $BUILD_DIRECTORY/build/lightgbmlib.jar $BUILD_ARTIFACTSTAGINGDIRECTORY/lightgbmlib_$OS_NAME.jar
cp ./build/lightgbmlib.jar $BUILD_ARTIFACTSTAGINGDIRECTORY/lightgbmlib_$OS_NAME.jar
fi
exit 0
fi
if [[ $TASK == "lint" ]]; then
cd ${BUILD_DIRECTORY}
mamba create -q -y -n $CONDA_ENV \
${CONDA_PYTHON_REQUIREMENT} \
'cmakelint>=1.4.2' \
@ -102,16 +106,16 @@ if [[ $TASK == "lint" ]]; then
'r-lintr>=3.1.2'
source activate $CONDA_ENV
echo "Linting Python code"
bash ${BUILD_DIRECTORY}/.ci/lint-python.sh || exit 1
bash ./.ci/lint-python.sh || exit 1
echo "Linting R code"
Rscript ${BUILD_DIRECTORY}/.ci/lint_r_code.R ${BUILD_DIRECTORY} || exit 1
Rscript ./.ci/lint_r_code.R "${BUILD_DIRECTORY}" || exit 1
echo "Linting C++ code"
bash ${BUILD_DIRECTORY}/.ci/lint-cpp.sh || exit 1
bash ./.ci/lint-cpp.sh || exit 1
exit 0
fi
if [[ $TASK == "check-docs" ]] || [[ $TASK == "check-links" ]]; then
cd $BUILD_DIRECTORY/docs
cd "${BUILD_DIRECTORY}/docs"
mamba env create \
-n $CONDA_ENV \
--file ./env.yml || exit 1
@ -123,9 +127,9 @@ if [[ $TASK == "check-docs" ]] || [[ $TASK == "check-links" ]]; then
'rstcheck>=6.2.0' || exit 1
source activate $CONDA_ENV
# check reStructuredText formatting
cd $BUILD_DIRECTORY/python-package
cd "${BUILD_DIRECTORY}/python-package"
rstcheck --report-level warning $(find . -type f -name "*.rst") || exit 1
cd $BUILD_DIRECTORY/docs
cd "${BUILD_DIRECTORY}/docs"
rstcheck --report-level warning --ignore-directives=autoclass,autofunction,autosummary,doxygenfile $(find . -type f -name "*.rst") || exit 1
# build docs
make html || exit 1
@ -136,11 +140,12 @@ if [[ $TASK == "check-docs" ]] || [[ $TASK == "check-links" ]]; then
exit 0
fi
# check the consistency of parameters' descriptions and other stuff
cp $BUILD_DIRECTORY/docs/Parameters.rst $BUILD_DIRECTORY/docs/Parameters-backup.rst
cp $BUILD_DIRECTORY/src/io/config_auto.cpp $BUILD_DIRECTORY/src/io/config_auto-backup.cpp
python $BUILD_DIRECTORY/helpers/parameter_generator.py || exit 1
diff $BUILD_DIRECTORY/docs/Parameters-backup.rst $BUILD_DIRECTORY/docs/Parameters.rst || exit 1
diff $BUILD_DIRECTORY/src/io/config_auto-backup.cpp $BUILD_DIRECTORY/src/io/config_auto.cpp || exit 1
cd "${BUILD_DIRECTORY}"
cp ./docs/Parameters.rst ./docs/Parameters-backup.rst
cp ./src/io/config_auto.cpp ./src/io/config_auto-backup.cpp
python ./helpers/parameter_generator.py || exit 1
diff ./docs/Parameters-backup.rst ./docs/Parameters.rst || exit 1
diff ./src/io/config_auto-backup.cpp ./src/io/config_auto.cpp || exit 1
exit 0
fi
@ -161,21 +166,21 @@ mamba create \
source activate $CONDA_ENV
cd $BUILD_DIRECTORY
cd "${BUILD_DIRECTORY}"
if [[ $TASK == "sdist" ]]; then
cd $BUILD_DIRECTORY && sh ./build-python.sh sdist || exit 1
sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit 1
pip install $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER.tar.gz -v || exit 1
sh ./build-python.sh sdist || exit 1
sh .ci/check_python_dists.sh ./dist || exit 1
pip install ./dist/lightgbm-$LGB_VER.tar.gz -v || exit 1
if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
cp $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER.tar.gz $BUILD_ARTIFACTSTAGINGDIRECTORY || exit 1
cp ./dist/lightgbm-$LGB_VER.tar.gz $BUILD_ARTIFACTSTAGINGDIRECTORY || exit 1
fi
pytest $BUILD_DIRECTORY/tests/python_package_test || exit 1
pytest ./tests/python_package_test || exit 1
exit 0
elif [[ $TASK == "bdist" ]]; then
if [[ $OS_NAME == "macos" ]]; then
cd $BUILD_DIRECTORY && sh ./build-python.sh bdist_wheel || exit 1
sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit 1
sh ./build-python.sh bdist_wheel || exit 1
sh .ci/check_python_dists.sh ./dist || exit 1
if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
cp dist/lightgbm-$LGB_VER-py3-none-macosx*.whl $BUILD_ARTIFACTSTAGINGDIRECTORY || exit 1
fi
@ -185,88 +190,88 @@ elif [[ $TASK == "bdist" ]]; then
else
PLATFORM="manylinux2014_$ARCH"
fi
cd $BUILD_DIRECTORY && sh ./build-python.sh bdist_wheel --integrated-opencl || exit 1
sh ./build-python.sh bdist_wheel --integrated-opencl || exit 1
mv \
./dist/*.whl \
./dist/tmp.whl || exit 1
mv \
./dist/tmp.whl \
./dist/lightgbm-$LGB_VER-py3-none-$PLATFORM.whl || exit 1
sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit 1
sh .ci/check_python_dists.sh ./dist || exit 1
if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
cp dist/lightgbm-$LGB_VER-py3-none-$PLATFORM.whl $BUILD_ARTIFACTSTAGINGDIRECTORY || exit 1
fi
# Make sure we can do both CPU and GPU; see tests/python_package_test/test_dual.py
export LIGHTGBM_TEST_DUAL_CPU_GPU=1
fi
pip install -v $BUILD_DIRECTORY/dist/*.whl || exit 1
pytest $BUILD_DIRECTORY/tests || exit 1
pip install -v ./dist/*.whl || exit 1
pytest ./tests || exit 1
exit 0
fi
if [[ $TASK == "gpu" ]]; then
sed -i'.bak' 's/std::string device_type = "cpu";/std::string device_type = "gpu";/' $BUILD_DIRECTORY/include/LightGBM/config.h
grep -q 'std::string device_type = "gpu"' $BUILD_DIRECTORY/include/LightGBM/config.h || exit 1 # make sure that changes were really done
sed -i'.bak' 's/std::string device_type = "cpu";/std::string device_type = "gpu";/' ./include/LightGBM/config.h
grep -q 'std::string device_type = "gpu"' ./include/LightGBM/config.h || exit 1 # make sure that changes were really done
if [[ $METHOD == "pip" ]]; then
cd $BUILD_DIRECTORY && sh ./build-python.sh sdist || exit 1
sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit 1
sh ./build-python.sh sdist || exit 1
sh .ci/check_python_dists.sh ./dist || exit 1
pip install \
-v \
--config-settings=cmake.define.USE_GPU=ON \
$BUILD_DIRECTORY/dist/lightgbm-$LGB_VER.tar.gz \
./dist/lightgbm-$LGB_VER.tar.gz \
|| exit 1
pytest $BUILD_DIRECTORY/tests/python_package_test || exit 1
pytest ./tests/python_package_test || exit 1
exit 0
elif [[ $METHOD == "wheel" ]]; then
cd $BUILD_DIRECTORY && sh ./build-python.sh bdist_wheel --gpu || exit 1
sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit 1
pip install $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER*.whl -v || exit 1
pytest $BUILD_DIRECTORY/tests || exit 1
sh ./build-python.sh bdist_wheel --gpu || exit 1
sh ./.ci/check_python_dists.sh ./dist || exit 1
pip install ./dist/lightgbm-$LGB_VER*.whl -v || exit 1
pytest ./tests || exit 1
exit 0
elif [[ $METHOD == "source" ]]; then
cmake -B build -S . -DUSE_GPU=ON
fi
elif [[ $TASK == "cuda" ]]; then
sed -i'.bak' 's/std::string device_type = "cpu";/std::string device_type = "cuda";/' $BUILD_DIRECTORY/include/LightGBM/config.h
grep -q 'std::string device_type = "cuda"' $BUILD_DIRECTORY/include/LightGBM/config.h || exit 1 # make sure that changes were really done
sed -i'.bak' 's/std::string device_type = "cpu";/std::string device_type = "cuda";/' ./include/LightGBM/config.h
grep -q 'std::string device_type = "cuda"' ./include/LightGBM/config.h || exit 1 # make sure that changes were really done
# by default ``gpu_use_dp=false`` for efficiency. change to ``true`` here for exact results in ci tests
sed -i'.bak' 's/gpu_use_dp = false;/gpu_use_dp = true;/' $BUILD_DIRECTORY/include/LightGBM/config.h
grep -q 'gpu_use_dp = true' $BUILD_DIRECTORY/include/LightGBM/config.h || exit 1 # make sure that changes were really done
sed -i'.bak' 's/gpu_use_dp = false;/gpu_use_dp = true;/' ./include/LightGBM/config.h
grep -q 'gpu_use_dp = true' ./include/LightGBM/config.h || exit 1 # make sure that changes were really done
if [[ $METHOD == "pip" ]]; then
cd $BUILD_DIRECTORY && sh ./build-python.sh sdist || exit 1
sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit 1
sh ./build-python.sh sdist || exit 1
sh ./.ci/check_python_dists.sh ./dist || exit 1
pip install \
-v \
--config-settings=cmake.define.USE_CUDA=ON \
$BUILD_DIRECTORY/dist/lightgbm-$LGB_VER.tar.gz \
./dist/lightgbm-$LGB_VER.tar.gz \
|| exit 1
pytest $BUILD_DIRECTORY/tests/python_package_test || exit 1
pytest ./tests/python_package_test || exit 1
exit 0
elif [[ $METHOD == "wheel" ]]; then
cd $BUILD_DIRECTORY && sh ./build-python.sh bdist_wheel --cuda || exit 1
sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit 1
pip install $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER*.whl -v || exit 1
pytest $BUILD_DIRECTORY/tests || exit 1
sh ./build-python.sh bdist_wheel --cuda || exit 1
sh ./.ci/check_python_dists.sh ./dist || exit 1
pip install ./dist/lightgbm-$LGB_VER*.whl -v || exit 1
pytest ./tests || exit 1
exit 0
elif [[ $METHOD == "source" ]]; then
cmake -B build -S . -DUSE_CUDA=ON
fi
elif [[ $TASK == "mpi" ]]; then
if [[ $METHOD == "pip" ]]; then
cd $BUILD_DIRECTORY && sh ./build-python.sh sdist || exit 1
sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit 1
sh ./build-python.sh sdist || exit 1
sh ./.ci/check_python_dists.sh ./dist || exit 1
pip install \
-v \
--config-settings=cmake.define.USE_MPI=ON \
$BUILD_DIRECTORY/dist/lightgbm-$LGB_VER.tar.gz \
./dist/lightgbm-$LGB_VER.tar.gz \
|| exit 1
pytest $BUILD_DIRECTORY/tests/python_package_test || exit 1
pytest ./tests/python_package_test || exit 1
exit 0
elif [[ $METHOD == "wheel" ]]; then
cd $BUILD_DIRECTORY && sh ./build-python.sh bdist_wheel --mpi || exit 1
sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit 1
pip install $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER*.whl -v || exit 1
pytest $BUILD_DIRECTORY/tests || exit 1
sh ./build-python.sh bdist_wheel --mpi || exit 1
sh ./.ci/check_python_dists.sh ./dist || exit 1
pip install ./dist/lightgbm-$LGB_VER*.whl -v || exit 1
pytest ./tests || exit 1
exit 0
elif [[ $METHOD == "source" ]]; then
cmake -B build -S . -DUSE_MPI=ON -DUSE_DEBUG=ON
@ -277,22 +282,22 @@ fi
cmake --build build --target _lightgbm -j4 || exit 1
cd $BUILD_DIRECTORY && sh ./build-python.sh install --precompile || exit 1
pytest $BUILD_DIRECTORY/tests || exit 1
sh ./build-python.sh install --precompile || exit 1
pytest ./tests || exit 1
if [[ $TASK == "regular" ]]; then
if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
if [[ $OS_NAME == "macos" ]]; then
cp $BUILD_DIRECTORY/lib_lightgbm.dylib $BUILD_ARTIFACTSTAGINGDIRECTORY/lib_lightgbm.dylib
cp ./lib_lightgbm.dylib $BUILD_ARTIFACTSTAGINGDIRECTORY/lib_lightgbm.dylib
else
if [[ $COMPILER == "gcc" ]]; then
objdump -T $BUILD_DIRECTORY/lib_lightgbm.so > $BUILD_DIRECTORY/objdump.log || exit 1
python $BUILD_DIRECTORY/helpers/check_dynamic_dependencies.py $BUILD_DIRECTORY/objdump.log || exit 1
objdump -T ./lib_lightgbm.so > ./objdump.log || exit 1
python ./helpers/check_dynamic_dependencies.py ./objdump.log || exit 1
fi
cp $BUILD_DIRECTORY/lib_lightgbm.so $BUILD_ARTIFACTSTAGINGDIRECTORY/lib_lightgbm.so
cp ./lib_lightgbm.so $BUILD_ARTIFACTSTAGINGDIRECTORY/lib_lightgbm.so
fi
fi
cd $BUILD_DIRECTORY/examples/python-guide
cd "$BUILD_DIRECTORY/examples/python-guide"
sed -i'.bak' '/import lightgbm as lgb/a\
import matplotlib\
matplotlib.use\(\"Agg\"\)\
@ -304,7 +309,7 @@ matplotlib.use\(\"Agg\"\)\
'ipywidgets>=8.1.2' \
'notebook>=7.1.2'
for f in *.py **/*.py; do python $f || exit 1; done # run all examples
cd $BUILD_DIRECTORY/examples/python-guide/notebooks
cd "$BUILD_DIRECTORY/examples/python-guide/notebooks"
sed -i'.bak' 's/INTERACTIVE = False/assert False, \\"Interactive mode disabled\\"/' interactive_plot_example.ipynb
jupyter nbconvert --ExecutePreprocessor.timeout=180 --to notebook --execute --inplace *.ipynb || exit 1 # run all notebooks

Просмотреть файл

@ -136,7 +136,7 @@ if [[ $OS_NAME == "macos" ]]; then
fi
Rscript --vanilla -e "options(install.packages.compile.from.source = '${compile_from_source}'); install.packages(${packages}, repos = '${CRAN_MIRROR}', lib = '${R_LIB_PATH}', dependencies = c('Depends', 'Imports', 'LinkingTo'), Ncpus = parallel::detectCores())" || exit 1
cd ${BUILD_DIRECTORY}
cd "${BUILD_DIRECTORY}"
PKG_TARBALL="lightgbm_*.tar.gz"
LOG_FILE_NAME="lightgbm.Rcheck/00check.log"
@ -147,7 +147,7 @@ elif [[ $R_BUILD_TYPE == "cran" ]]; then
# on Linux, we recreate configure in CI to test if
# a change in a PR has changed configure.ac
if [[ $OS_NAME == "linux" ]]; then
${BUILD_DIRECTORY}/R-package/recreate-configure.sh
./R-package/recreate-configure.sh
num_files_changed=$(
git diff --name-only | wc -l

Просмотреть файл

@ -10,7 +10,7 @@ $env:CONDA_ENV = "test-env"
$env:LGB_VER = (Get-Content $env:BUILD_SOURCESDIRECTORY\VERSION.txt).trim()
if ($env:TASK -eq "r-package") {
& $env:BUILD_SOURCESDIRECTORY\.ci\test_r_package_windows.ps1 ; Check-Output $?
& .\.ci\test_r_package_windows.ps1 ; Check-Output $?
Exit 0
}
@ -31,7 +31,7 @@ if ($env:TASK -eq "swig") {
cmake -B build -S . -A x64 -DUSE_SWIG=ON ; Check-Output $?
cmake --build build --target ALL_BUILD --config Release ; Check-Output $?
if ($env:AZURE -eq "true") {
cp $env:BUILD_SOURCESDIRECTORY/build/lightgbmlib.jar $env:BUILD_ARTIFACTSTAGINGDIRECTORY/lightgbmlib_win.jar ; Check-Output $?
cp ./build/lightgbmlib.jar $env:BUILD_ARTIFACTSTAGINGDIRECTORY/lightgbmlib_win.jar ; Check-Output $?
}
Exit 0
}
@ -60,18 +60,17 @@ if ($env:TASK -ne "bdist") {
conda activate $env:CONDA_ENV
}
cd $env:BUILD_SOURCESDIRECTORY
if ($env:TASK -eq "regular") {
cmake -B build -S . -A x64 ; Check-Output $?
cmake --build build --target ALL_BUILD --config Release ; Check-Output $?
cd $env:BUILD_SOURCESDIRECTORY
sh $env:BUILD_SOURCESDIRECTORY/build-python.sh install --precompile ; Check-Output $?
cp $env:BUILD_SOURCESDIRECTORY/Release/lib_lightgbm.dll $env:BUILD_ARTIFACTSTAGINGDIRECTORY
cp $env:BUILD_SOURCESDIRECTORY/Release/lightgbm.exe $env:BUILD_ARTIFACTSTAGINGDIRECTORY
sh ./build-python.sh install --precompile ; Check-Output $?
cp ./Release/lib_lightgbm.dll $env:BUILD_ARTIFACTSTAGINGDIRECTORY
cp ./Release/lightgbm.exe $env:BUILD_ARTIFACTSTAGINGDIRECTORY
}
elseif ($env:TASK -eq "sdist") {
cd $env:BUILD_SOURCESDIRECTORY
sh $env:BUILD_SOURCESDIRECTORY/build-python.sh sdist ; Check-Output $?
sh $env:BUILD_SOURCESDIRECTORY/.ci/check_python_dists.sh $env:BUILD_SOURCESDIRECTORY/dist ; Check-Output $?
sh ./build-python.sh sdist ; Check-Output $?
sh ./.ci/check_python_dists.sh ./dist ; Check-Output $?
cd dist; pip install @(Get-ChildItem *.gz) -v ; Check-Output $?
}
elseif ($env:TASK -eq "bdist") {
@ -85,17 +84,15 @@ elseif ($env:TASK -eq "bdist") {
Get-ItemProperty -Path Registry::HKEY_LOCAL_MACHINE\SOFTWARE\Khronos\OpenCL\Vendors
conda activate $env:CONDA_ENV
cd $env:BUILD_SOURCESDIRECTORY
sh "build-python.sh" bdist_wheel --integrated-opencl ; Check-Output $?
sh $env:BUILD_SOURCESDIRECTORY/.ci/check_python_dists.sh $env:BUILD_SOURCESDIRECTORY/dist ; Check-Output $?
sh ./.ci/check_python_dists.sh ./dist ; Check-Output $?
cd dist; pip install @(Get-ChildItem *py3-none-win_amd64.whl) ; Check-Output $?
cp @(Get-ChildItem *py3-none-win_amd64.whl) $env:BUILD_ARTIFACTSTAGINGDIRECTORY
} elseif (($env:APPVEYOR -eq "true") -and ($env:TASK -eq "python")) {
cd $env:BUILD_SOURCESDIRECTORY
if ($env:COMPILER -eq "MINGW") {
sh $env:BUILD_SOURCESDIRECTORY/build-python.sh install --mingw ; Check-Output $?
sh ./build-python.sh install --mingw ; Check-Output $?
} else {
sh $env:BUILD_SOURCESDIRECTORY/build-python.sh install; Check-Output $?
sh ./build-python.sh install; Check-Output $?
}
}