From 30942a32e343d1676a15a2811b002c5aae1853d0 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 15 Jun 2023 11:30:34 -0500 Subject: [PATCH] [python-package] [ci] switch to PEP 517 / 518 builds (remove `setup.py`) (fixes #5061) (#5759) --- .ci/test.sh | 18 +- .ci/test_windows.ps1 | 4 +- CMakeLists.txt | 4 + build-python.sh | 146 +++++++----- python-package/MANIFEST.in | 47 ---- python-package/README.rst | 55 +++-- python-package/lightgbm/libpath.py | 9 +- python-package/pyproject.toml | 60 +++++ python-package/setup.cfg | 17 ++ python-package/setup.py | 355 ----------------------------- 10 files changed, 224 insertions(+), 491 deletions(-) delete mode 100644 python-package/MANIFEST.in delete mode 100644 python-package/setup.py diff --git a/.ci/test.sh b/.ci/test.sh index 80ed7d2d0..665e7f654 100755 --- a/.ci/test.sh +++ b/.ci/test.sh @@ -159,6 +159,9 @@ elif [[ $TASK == "bdist" ]]; then sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit -1 mv \ ./dist/*.whl \ + ./dist/tmp.whl || exit -1 + mv \ + ./dist/tmp.whl \ dist/lightgbm-$LGB_VER-py3-none-macosx_10_15_x86_64.macosx_11_6_x86_64.macosx_12_5_x86_64.whl || exit -1 if [[ $PRODUCES_ARTIFACTS == "true" ]]; then cp dist/lightgbm-$LGB_VER-py3-none-macosx*.whl $BUILD_ARTIFACTSTAGINGDIRECTORY || exit -1 @@ -173,6 +176,9 @@ elif [[ $TASK == "bdist" ]]; then cd $BUILD_DIRECTORY && sh ./build-python.sh bdist_wheel --integrated-opencl || exit -1 mv \ ./dist/*.whl \ + ./dist/tmp.whl || exit -1 + mv \ + ./dist/tmp.whl \ ./dist/lightgbm-$LGB_VER-py3-none-$PLATFORM.whl || exit -1 sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit -1 if [[ $PRODUCES_ARTIFACTS == "true" ]]; then @@ -186,12 +192,6 @@ elif [[ $TASK == "bdist" ]]; then exit 0 fi -# temporarily pin pip to versions that support 'pip install --install-option' -# ref: https://github.com/microsoft/LightGBM/issues/5061#issuecomment-1510642287 -if [[ $METHOD == "pip" ]]; then - pip install 'pip<23.1' -fi - if [[ $TASK == "gpu" ]]; then sed -i'.bak' 's/std::string device_type = "cpu";/std::string device_type = "gpu";/' $BUILD_DIRECTORY/include/LightGBM/config.h grep -q 'std::string device_type = "gpu"' $BUILD_DIRECTORY/include/LightGBM/config.h || exit -1 # make sure that changes were really done @@ -201,7 +201,7 @@ if [[ $TASK == "gpu" ]]; then pip install \ --user \ -v \ - --install-option=--gpu \ + --config-settings=cmake.define.USE_GPU=ON \ $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER.tar.gz \ || exit -1 pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1 @@ -229,7 +229,7 @@ elif [[ $TASK == "cuda" ]]; then pip install \ --user \ -v \ - --install-option=--cuda \ + --config-settings=cmake.define.USE_CUDA=ON \ $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER.tar.gz \ || exit -1 pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1 @@ -252,7 +252,7 @@ elif [[ $TASK == "mpi" ]]; then pip install \ --user \ -v \ - --install-option=--mpi \ + --config-settings=cmake.define.USE_MPI=ON \ $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER.tar.gz \ || exit -1 pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1 diff --git a/.ci/test_windows.ps1 b/.ci/test_windows.ps1 index 80e0bb220..5962a9441 100644 --- a/.ci/test_windows.ps1 +++ b/.ci/test_windows.ps1 @@ -96,8 +96,8 @@ elseif ($env:TASK -eq "bdist") { cd $env:BUILD_SOURCESDIRECTORY sh "build-python.sh" bdist_wheel --integrated-opencl ; Check-Output $? sh $env:BUILD_SOURCESDIRECTORY/.ci/check_python_dists.sh $env:BUILD_SOURCESDIRECTORY/dist ; Check-Output $? - cd dist; pip install --user @(Get-ChildItem *.whl) ; Check-Output $? - cp @(Get-ChildItem *.whl) $env:BUILD_ARTIFACTSTAGINGDIRECTORY + cd dist; pip install --user @(Get-ChildItem *py3-none-win_amd64.whl) ; Check-Output $? + cp @(Get-ChildItem *py3-none-win_amd64.whl) $env:BUILD_ARTIFACTSTAGINGDIRECTORY } elseif (($env:APPVEYOR -eq "true") -and ($env:TASK -eq "python")) { cd $env:BUILD_SOURCESDIRECTORY if ($env:COMPILER -eq "MINGW") { diff --git a/CMakeLists.txt b/CMakeLists.txt index 0792f0959..5087d6a8f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -645,6 +645,10 @@ if(BUILD_CLI) ) endif() +if(__BUILD_FOR_PYTHON) + set(CMAKE_INSTALL_PREFIX "lightgbm") +endif() + install( TARGETS _lightgbm RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX}/bin diff --git a/build-python.sh b/build-python.sh index 505d61e22..a60c3b5e8 100755 --- a/build-python.sh +++ b/build-python.sh @@ -94,75 +94,79 @@ while [ $# -gt 0 ]; do then shift; fi BOOST_DIR="${1#*=}" - BUILD_ARGS="${BUILD_ARGS} --boost-dir='${BOOST_DIR}'" + BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.Boost_DIR='${BOOST_DIR}'" ;; --boost-include-dir|--boost-include-dir=*) if [[ "$1" != *=* ]]; then shift; fi BOOST_INCLUDE_DIR="${1#*=}" - BUILD_ARGS="${BUILD_ARGS} --boost-include-dir='${BOOST_INCLUDE_DIR}'" + BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.Boost_INCLUDE_DIR='${BOOST_INCLUDE_DIR}'" ;; --boost-librarydir|--boost-librarydir=*) if [[ "$1" != *=* ]]; then shift; fi BOOST_LIBRARY_DIR="${1#*=}" - BUILD_ARGS="${BUILD_ARGS} --boost-librarydir='${BOOST_LIBRARY_DIR}'" + BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.BOOST_LIBRARYDIR='${BOOST_LIBRARY_DIR}'" ;; --boost-root|--boost-root=*) if [[ "$1" != *=* ]]; then shift; fi BOOST_ROOT="${1#*=}" - BUILD_ARGS="${BUILD_ARGS} --boost-root='${BOOST_ROOT}'" + BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.Boost_ROOT='${BOOST_ROOT}'" ;; --opencl-include-dir|--opencl-include-dir=*) if [[ "$1" != *=* ]]; then shift; fi OPENCL_INCLUDE_DIR="${1#*=}" - BUILD_ARGS="${BUILD_ARGS} --opencl-include-dir='${OPENCL_INCLUDE_DIR}'" + BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.OpenCL_INCLUDE_DIR='${OPENCL_INCLUDE_DIR}'" ;; --opencl-library|--opencl-library=*) if [[ "$1" != *=* ]]; then shift; fi OPENCL_LIBRARY="${1#*=}" - BUILD_ARGS="${BUILD_ARGS} --opencl-library='${OPENCL_LIBRARY}'" + BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.OpenCL_LIBRARY='${OPENCL_LIBRARY}'" ;; ######### # flags # ######### --bit32) - BUILD_ARGS="${BUILD_ARGS} --bit32" + export CMAKE_GENERATOR="Visual Studio 17 2022" + export CMAKE_GENERATOR_PLATFORM="Win32" + echo "[INFO] Attempting to build 32-bit version of LightGBM, which is only supported on Windows with generator '${CMAKE_GENERATOR}'." ;; --cuda) - BUILD_ARGS="${BUILD_ARGS} --cuda" + BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.USE_CUDA=ON" ;; --gpu) - BUILD_ARGS="${BUILD_ARGS} --gpu" + BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.USE_GPU=ON" ;; --hdfs) - BUILD_ARGS="${BUILD_ARGS} --hdfs" + BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.USE_HDFS=ON" ;; --integrated-opencl) - BUILD_ARGS="${BUILD_ARGS} --integrated-opencl" + BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.__INTEGRATE_OPENCL=ON" ;; --mingw) - BUILD_ARGS="${BUILD_ARGS} --mingw" + export CMAKE_GENERATOR='MinGW Makefiles' + # ref: https://stackoverflow.com/a/45104058/3986677 + BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.CMAKE_SH=CMAKE_SH-NOTFOUND" ;; --mpi) - BUILD_ARGS="${BUILD_ARGS} --mpi" + BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.USE_MPI=ON" ;; --nomp) - BUILD_ARGS="${BUILD_ARGS} --nomp" + BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.USE_OPENMP=OFF" ;; --precompile) PRECOMPILE="true" ;; --time-costs) - BUILD_ARGS="${PIP_INSTALL_ARGS} --time-costs" + BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.USE_TIMETAG=ON" ;; --user) PIP_INSTALL_ARGS="${PIP_INSTALL_ARGS} --user" @@ -175,6 +179,8 @@ while [ $# -gt 0 ]; do shift done +pip install --prefer-binary 'build>=0.10.0' + # create a new directory that just contains the files needed # to build the Python package create_isolated_source_dir() { @@ -189,21 +195,15 @@ create_isolated_source_dir() { cp -R ./python-package ./lightgbm-python - # temporarily remove these files until - # https://github.com/microsoft/LightGBM/issues/5061 is done - rm ./lightgbm-python/pyproject.toml - rm ./lightgbm-python/setup.cfg - cp LICENSE ./lightgbm-python/ cp VERSION.txt ./lightgbm-python/lightgbm/VERSION.txt - mkdir -p ./lightgbm-python/compile - cp -R ./cmake ./lightgbm-python/compile - cp CMakeLists.txt ./lightgbm-python/compile - cp -R ./include ./lightgbm-python/compile - cp -R ./src ./lightgbm-python/compile - cp -R ./swig ./lightgbm-python/compile - cp -R ./windows ./lightgbm-python/compile + cp -R ./cmake ./lightgbm-python + cp CMakeLists.txt ./lightgbm-python + cp -R ./include ./lightgbm-python + cp -R ./src ./lightgbm-python + cp -R ./swig ./lightgbm-python + cp -R ./windows ./lightgbm-python # include only specific files from external_libs, to keep the package # small and avoid redistributing code with licenses incompatible with @@ -212,77 +212,77 @@ create_isolated_source_dir() { ###################### # fast_double_parser # ###################### - mkdir -p ./lightgbm-python/compile/external_libs/fast_double_parser + mkdir -p ./lightgbm-python/external_libs/fast_double_parser cp \ external_libs/fast_double_parser/CMakeLists.txt \ - ./lightgbm-python/compile/external_libs/fast_double_parser/CMakeLists.txt + ./lightgbm-python/external_libs/fast_double_parser/CMakeLists.txt cp \ external_libs/fast_double_parser/LICENSE* \ - ./lightgbm-python/compile/external_libs/fast_double_parser/ + ./lightgbm-python/external_libs/fast_double_parser/ - mkdir -p ./lightgbm-python/compile/external_libs/fast_double_parser/include/ + mkdir -p ./lightgbm-python/external_libs/fast_double_parser/include/ cp \ external_libs/fast_double_parser/include/fast_double_parser.h \ - ./lightgbm-python/compile/external_libs/fast_double_parser/include/ + ./lightgbm-python/external_libs/fast_double_parser/include/ ####### # fmt # ####### - mkdir -p ./lightgbm-python/compile/external_libs/fmt + mkdir -p ./lightgbm-python/external_libs/fmt cp \ external_libs/fast_double_parser/CMakeLists.txt \ - ./lightgbm-python/compile/external_libs/fmt/CMakeLists.txt + ./lightgbm-python/external_libs/fmt/CMakeLists.txt cp \ external_libs/fmt/LICENSE* \ - ./lightgbm-python/compile/external_libs/fmt/ + ./lightgbm-python/external_libs/fmt/ - mkdir -p ./lightgbm-python/compile/external_libs/fmt/include/fmt + mkdir -p ./lightgbm-python/external_libs/fmt/include/fmt cp \ external_libs/fmt/include/fmt/*.h \ - ./lightgbm-python/compile/external_libs/fmt/include/fmt/ + ./lightgbm-python/external_libs/fmt/include/fmt/ ######### # Eigen # ######### - mkdir -p ./lightgbm-python/compile/external_libs/eigen/Eigen + mkdir -p ./lightgbm-python/external_libs/eigen/Eigen cp \ external_libs/eigen/CMakeLists.txt \ - ./lightgbm-python/compile/external_libs/eigen/CMakeLists.txt + ./lightgbm-python/external_libs/eigen/CMakeLists.txt modules="Cholesky Core Dense Eigenvalues Geometry Householder Jacobi LU QR SVD" for eigen_module in ${modules}; do cp \ external_libs/eigen/Eigen/${eigen_module} \ - ./lightgbm-python/compile/external_libs/eigen/Eigen/${eigen_module} + ./lightgbm-python/external_libs/eigen/Eigen/${eigen_module} if [ ${eigen_module} != "Dense" ]; then - mkdir -p ./lightgbm-python/compile/external_libs/eigen/Eigen/src/${eigen_module}/ + mkdir -p ./lightgbm-python/external_libs/eigen/Eigen/src/${eigen_module}/ cp \ -R \ external_libs/eigen/Eigen/src/${eigen_module}/* \ - ./lightgbm-python/compile/external_libs/eigen/Eigen/src/${eigen_module}/ + ./lightgbm-python/external_libs/eigen/Eigen/src/${eigen_module}/ fi done - mkdir -p ./lightgbm-python/compile/external_libs/eigen/Eigen/misc + mkdir -p ./lightgbm-python/external_libs/eigen/Eigen/misc cp \ -R \ external_libs/eigen/Eigen/src/misc \ - ./lightgbm-python/compile/external_libs/eigen/Eigen/src/misc/ + ./lightgbm-python/external_libs/eigen/Eigen/src/misc/ - mkdir -p ./lightgbm-python/compile/external_libs/eigen/Eigen/plugins + mkdir -p ./lightgbm-python/external_libs/eigen/Eigen/plugins cp \ -R \ external_libs/eigen/Eigen/src/plugins \ - ./lightgbm-python/compile/external_libs/eigen/Eigen/src/plugins/ + ./lightgbm-python/external_libs/eigen/Eigen/src/plugins/ ################### # compute (Boost) # ################### - mkdir -p ./lightgbm-python/compile/external_libs/compute + mkdir -p ./lightgbm-python/external_libs/compute cp \ -R \ external_libs/compute/include \ - ./lightgbm-python/compile/external_libs/compute/include/ + ./lightgbm-python/external_libs/compute/include/ } create_isolated_source_dir @@ -292,9 +292,39 @@ cd ./lightgbm-python # installation involves building the wheel + `pip install`-ing it if test "${INSTALL}" = true; then if test "${PRECOMPILE}" = true; then - echo "--- installing lightgbm (from precompiled lib_lightgbm) ---" - python setup.py install ${PIP_INSTALL_ARGS} --precompile - exit 0 + BUILD_SDIST=true + BUILD_WHEEL=false + BUILD_ARGS="" + rm -rf \ + ./cmake \ + ./CMakeLists.txt \ + ./external_libs \ + ./include \ + ./src \ + ./swig \ + ./windows + # use regular-old setuptools for these builds, to avoid + # trying to recompile the shared library + sed -i.bak -e '/start:build-system/,/end:build-system/d' pyproject.toml + echo '[build-system]' >> ./pyproject.toml + echo 'requires = ["setuptools"]' >> ./pyproject.toml + echo 'build-backend = "setuptools.build_meta"' >> ./pyproject.toml + echo "" >> ./pyproject.toml + echo "recursive-include lightgbm *.dll *.so" > ./MANIFEST.in + echo "" >> ./MANIFEST.in + mkdir -p ./lightgbm/lib + if test -f ../lib_lightgbm.so; then + echo "found pre-compiled lib_lightgbm.so" + cp ../lib_lightgbm.so ./lightgbm/lib/lib_lightgbm.so + elif test -f ../Release/lib_lightgbm.dll; then + echo "found pre-compiled Release/lib_lightgbm.dll" + cp ../Release/lib_lightgbm.dll ./lightgbm/lib/lib_lightgbm.dll + elif test -f ../windows/x64/DLL/lib_lightgbm.dll; then + echo "found pre-compiled windows/x64/DLL/lib_lightgbm.dll" + cp ../windows/x64/DLL/lib_lightgbm.dll ./lightgbm/lib/lib_lightgbm.dll + cp ../windows/x64/DLL/lib_lightgbm.lib ./lightgbm/lib/lib_lightgbm.lib + fi + rm -f ./*.bak else BUILD_SDIST="false" BUILD_WHEEL="true" @@ -304,16 +334,20 @@ fi if test "${BUILD_SDIST}" = true; then echo "--- building sdist ---" rm -f ../dist/*.tar.gz - python ./setup.py sdist \ - --dist-dir ../dist + python -m build \ + --sdist \ + --outdir ../dist \ + . fi if test "${BUILD_WHEEL}" = true; then echo "--- building wheel ---" rm -f ../dist/*.whl || true - python setup.py bdist_wheel \ - --dist-dir ../dist \ - ${BUILD_ARGS} + python -m build \ + --wheel \ + --outdir ../dist \ + ${BUILD_ARGS} \ + . fi if test "${INSTALL}" = true; then diff --git a/python-package/MANIFEST.in b/python-package/MANIFEST.in deleted file mode 100644 index 0effeba8e..000000000 --- a/python-package/MANIFEST.in +++ /dev/null @@ -1,47 +0,0 @@ -prune build -include LICENSE -include *.rst *.txt -recursive-include lightgbm VERSION.txt py.typed *.py *.so -include compile/CMakeLists.txt -include compile/cmake/IntegratedOpenCL.cmake -recursive-include compile *.so -recursive-include compile/Release *.dll -include compile/external_libs/compute/CMakeLists.txt -recursive-include compile/external_libs/compute/cmake * -recursive-include compile/external_libs/compute/include * -recursive-include compile/external_libs/compute/meta * -include compile/external_libs/eigen/CMakeLists.txt -include compile/external_libs/eigen/Eigen/Cholesky -include compile/external_libs/eigen/Eigen/Core -include compile/external_libs/eigen/Eigen/Dense -include compile/external_libs/eigen/Eigen/Eigenvalues -include compile/external_libs/eigen/Eigen/Geometry -include compile/external_libs/eigen/Eigen/Householder -include compile/external_libs/eigen/Eigen/Jacobi -include compile/external_libs/eigen/Eigen/LU -include compile/external_libs/eigen/Eigen/QR -include compile/external_libs/eigen/Eigen/SVD -recursive-include compile/external_libs/eigen/Eigen/src/Cholesky * -recursive-include compile/external_libs/eigen/Eigen/src/Core * -recursive-include compile/external_libs/eigen/Eigen/src/Eigenvalues * -recursive-include compile/external_libs/eigen/Eigen/src/Geometry * -recursive-include compile/external_libs/eigen/Eigen/src/Householder * -recursive-include compile/external_libs/eigen/Eigen/src/Jacobi * -recursive-include compile/external_libs/eigen/Eigen/src/LU * -recursive-include compile/external_libs/eigen/Eigen/src/misc * -recursive-include compile/external_libs/eigen/Eigen/src/plugins * -recursive-include compile/external_libs/eigen/Eigen/src/QR * -recursive-include compile/external_libs/eigen/Eigen/src/SVD * -include compile/external_libs/fast_double_parser/CMakeLists.txt -include compile/external_libs/fast_double_parser/LICENSE -include compile/external_libs/fast_double_parser/LICENSE.BSL -recursive-include compile/external_libs/fast_double_parser/include * -include compile/external_libs/fmt/CMakeLists.txt -include compile/external_libs/fmt/LICENSE.rst -recursive-include compile/external_libs/fmt/include * -recursive-include compile/include * -recursive-include compile/src * -recursive-include compile/windows LightGBM.sln LightGBM.vcxproj -recursive-include compile/windows/x64/DLL *.dll -global-exclude *.py[co] -exclude compile/external_libs/compute/.git diff --git a/python-package/README.rst b/python-package/README.rst index 2f927f4f2..ee3432042 100644 --- a/python-package/README.rst +++ b/python-package/README.rst @@ -60,7 +60,7 @@ Build Threadless Version .. code:: sh - pip install lightgbm --install-option=--nomp + pip install lightgbm --config-settings=cmake.define.USE_OPENMP=OFF All requirements, except the **OpenMP** requirement, from `Build from Sources section <#build-from-sources>`__ apply for this installation option as well. @@ -71,7 +71,7 @@ Build MPI Version .. code:: sh - pip install lightgbm --install-option=--mpi + pip install lightgbm --config-settings=cmake.define.USE_MPI=ON All requirements from `Build from Sources section <#build-from-sources>`__ apply for this installation option as well. @@ -84,7 +84,7 @@ Build GPU Version .. code:: sh - pip install lightgbm --install-option=--gpu + pip install lightgbm --config-settings=cmake.define.USE_GPU=ON All requirements from `Build from Sources section <#build-from-sources>`__ apply for this installation option as well. @@ -94,21 +94,24 @@ For **Windows** users, `CMake`_ (version 3.8 or higher) is strongly required. .. code:: sh - pip install lightgbm --install-option=--gpu --install-option="--opencl-include-dir=/usr/local/cuda/include/" --install-option="--opencl-library=/usr/local/cuda/lib64/libOpenCL.so" + pip install lightgbm \ + --config-settings=cmake.define.USE_GPU=ON \ + --config-settings=cmake.define.OpenCL_INCLUDE_DIR="/usr/local/cuda/include/" \ + --config-settings=cmake.define.OpenCL_LIBRARY="/usr/local/cuda/lib64/libOpenCL.so" -All available options: +All available options that can be passed via ``cmake.define.{option}``. -- boost-root +- Boost_ROOT -- boost-dir +- Boost_DIR -- boost-include-dir +- Boost_INCLUDE_DIR -- boost-librarydir +- BOOST_LIBRARYDIR -- opencl-include-dir +- OpenCL_INCLUDE_DIR -- opencl-library +- OpenCL_LIBRARY For more details see `FindBoost `__ and `FindOpenCL `__. @@ -117,7 +120,7 @@ Build CUDA Version .. code:: sh - pip install lightgbm --install-option=--cuda + pip install lightgbm --config-settings=cmake.define.USE_CUDA=ON All requirements from `Build from Sources section <#build-from-sources>`__ apply for this installation option as well, and `CMake`_ (version 3.16 or higher) is strongly required. @@ -130,7 +133,7 @@ Build HDFS Version .. code:: sh - pip install lightgbm --install-option=--hdfs + pip install lightgbm --config-settings=cmake.define.USE_HDFS=ON All requirements from `Build from Sources section <#build-from-sources>`__ apply for this installation option as well. @@ -143,7 +146,9 @@ Build with MinGW-w64 on Windows .. code:: sh - pip install lightgbm --install-option=--mingw + # in sh.exe, git bash, or other Unix-like shell + export CMAKE_GENERATOR='MinGW Makefiles' + pip install lightgbm --config-settings=cmake.define.CMAKE_SH=CMAKE_SH-NOTFOUND `CMake`_ and `MinGW-w64 `_ should be installed first. @@ -155,7 +160,10 @@ Build 32-bit Version with 32-bit Python .. code:: sh - pip install lightgbm --install-option=--bit32 + # in sh.exe, git bash, or other Unix-like shell + export CMAKE_GENERATOR='Visual Studio 17 2022' + export CMAKE_GENERATOR_PLATFORM='Win32' + pip install --no-binary lightgbm lightgbm By default, installation in environment with 32-bit Python is prohibited. However, you can remove this prohibition on your own risk by passing ``bit32`` option. @@ -166,7 +174,7 @@ Build with Time Costs Output .. code:: sh - pip install lightgbm --install-option=--time-costs + pip install lightgbm --config-settings=cmake.define.USE_TIMETAG=ON Use this option to make LightGBM output time costs for different internal routines, to investigate and benchmark its performance. @@ -221,6 +229,21 @@ Build Wheel File You can use ``sh ./build-python.sh install bdist_wheel`` instead of ``sh ./build-python.sh install`` to build wheel file and use it for installation later. This might be useful for systems with restricted or completely without network access. +Build With MSBuild +****************** + +To use ``MSBuild`` (Windows-only), first build ``lib_lightgbm.dll`` by running the following from the root of the repo. + +.. code:: sh + + MSBuild.exe windows/LightGBM.sln /p:Configuration=DLL /p:Platform=x64 /p:PlatformToolset=v143 + +Then install the Python package using that library. + +.. code:: sh + + sh ./build-python.sh install --precompile + Install Dask-package '''''''''''''''''''' diff --git a/python-package/lightgbm/libpath.py b/python-package/lightgbm/libpath.py index 05dfad1da..c096a6f1b 100644 --- a/python-package/lightgbm/libpath.py +++ b/python-package/lightgbm/libpath.py @@ -15,15 +15,12 @@ def find_lib_path() -> List[str]: lib_path: list of str List of all found library paths to LightGBM. """ - curr_path = Path(__file__).absolute().parent + curr_path = Path(__file__).absolute() dll_path = [curr_path, curr_path.parents[1], - curr_path / 'compile', - curr_path.parent / 'compile', - curr_path.parents[1] / 'lib'] + curr_path.parents[0] / 'bin', + curr_path.parents[0] / 'lib'] if system() in ('Windows', 'Microsoft'): - dll_path.append(curr_path.parent / 'compile' / 'Release') - dll_path.append(curr_path.parent / 'compile' / 'windows' / 'x64' / 'DLL') dll_path.append(curr_path.parents[1] / 'Release') dll_path.append(curr_path.parents[1] / 'windows' / 'x64' / 'DLL') dll_path = [p / 'lib_lightgbm.dll' for p in dll_path] diff --git a/python-package/pyproject.toml b/python-package/pyproject.toml index c3d550047..2962b4a4d 100644 --- a/python-package/pyproject.toml +++ b/python-package/pyproject.toml @@ -1,3 +1,63 @@ +[project] + +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Natural Language :: English", + "Operating System :: MacOS", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX", + "Operating System :: Unix", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Topic :: Scientific/Engineering :: Artificial Intelligence" +] +description = "LightGBM Python Package" +license = {file = "LICENSE"} +maintainers = [ + {name = "Yu Shi", email = "yushi@microsoft.com"} +] +name = "lightgbm" +readme = "README.rst" +requires-python = ">=3.6" +version = "3.3.5.99" + +[project.urls] +homepage = "https://github.com/microsoft/LightGBM" +documentation = "https://lightgbm.readthedocs.io/en/latest/" +repository = "https://github.com/microsoft/LightGBM.git" +changelog = "https://github.com/microsoft/LightGBM/releases" + +# start:build-system +[build-system] + +requires = ["scikit-build-core>=0.4.4"] +build-backend = "scikit_build_core.build" + +# based on https://github.com/scikit-build/scikit-build-core#configuration +[tool.scikit-build] + +cmake.minimum-version = "3.15" +ninja.minimum-version = "1.11" +ninja.make-fallback = true +cmake.args = [ + "-D__BUILD_FOR_PYTHON:BOOL=ON" +] +cmake.verbose = false +cmake.build-type = "Release" +logging.level = "INFO" +sdist.reproducible = true +wheel.py-api = "py3" +experimental = false +strict-config = true +minimum-version = "0.4.4" + +# end:build-system + [tool.isort] line_length = 120 skip_glob = [ diff --git a/python-package/setup.cfg b/python-package/setup.cfg index 0f2746df1..2b583a0cf 100644 --- a/python-package/setup.cfg +++ b/python-package/setup.cfg @@ -1,3 +1,20 @@ +[options] +include_package_data = True +install_requires = + numpy + scikit-learn!=0.22.0 + scipy + +[options.extras_require] +dask = + dask[array]>=2.0.0 + dask[dataframe]>=2.0.0 + dask[distributed]>=2.0.0 + pandas + +[options.packages.find] +where = lightgbm + [flake8] ignore = # line too long diff --git a/python-package/setup.py b/python-package/setup.py deleted file mode 100644 index 565cddd75..000000000 --- a/python-package/setup.py +++ /dev/null @@ -1,355 +0,0 @@ -# coding: utf-8 -"""Setup lightgbm package.""" -import logging -import struct -import subprocess -import sys -from os import chdir -from pathlib import Path -from platform import system -from shutil import rmtree -from typing import List, Optional - -from setuptools import find_packages, setup -from setuptools.command.install import install -from setuptools.command.install_lib import install_lib -from setuptools.command.sdist import sdist -from wheel.bdist_wheel import bdist_wheel - -LIGHTGBM_OPTIONS = [ - ('mingw', 'm', 'Compile with MinGW'), - ('integrated-opencl', None, 'Compile integrated OpenCL version'), - ('gpu', 'g', 'Compile GPU version'), - ('cuda', None, 'Compile CUDA version'), - ('mpi', None, 'Compile MPI version'), - ('nomp', None, 'Compile version without OpenMP support'), - ('hdfs', 'h', 'Compile HDFS version'), - ('bit32', None, 'Compile 32-bit version'), - ('precompile', 'p', 'Use precompiled library'), - ('time-costs', None, 'Output time costs for different internal routines'), - ('boost-root=', None, 'Boost preferred installation prefix'), - ('boost-dir=', None, 'Directory with Boost package configuration file'), - ('boost-include-dir=', None, 'Directory containing Boost headers'), - ('boost-librarydir=', None, 'Preferred Boost library directory'), - ('opencl-include-dir=', None, 'OpenCL include directory'), - ('opencl-library=', None, 'Path to OpenCL library') -] - - -def find_lib() -> List[str]: - libpath_py = CURRENT_DIR / 'lightgbm' / 'libpath.py' - libpath = {'__file__': libpath_py} - exec(compile(libpath_py.read_bytes(), libpath_py, 'exec'), libpath, libpath) - - LIB_PATH = libpath['find_lib_path']() # type: ignore - logger.info(f"Installing lib_lightgbm from: {LIB_PATH}") - return LIB_PATH - - -def clear_path(path: Path) -> None: - if path.is_dir(): - for file_name in path.iterdir(): - if file_name.is_dir(): - rmtree(file_name) - else: - file_name.unlink() - - -def silent_call(cmd: List[str], raise_error: bool = False, error_msg: str = '') -> int: - try: - with open(LOG_PATH, "ab") as log: - subprocess.check_call(cmd, stderr=log, stdout=log) - return 0 - except Exception: - if raise_error: - raise Exception("\n".join((error_msg, LOG_NOTICE))) - return 1 - - -def compile_cpp( - use_mingw: bool = False, - use_gpu: bool = False, - use_cuda: bool = False, - use_mpi: bool = False, - use_hdfs: bool = False, - boost_root: Optional[str] = None, - boost_dir: Optional[str] = None, - boost_include_dir: Optional[str] = None, - boost_librarydir: Optional[str] = None, - opencl_include_dir: Optional[str] = None, - opencl_library: Optional[str] = None, - nomp: bool = False, - bit32: bool = False, - integrated_opencl: bool = False, - time_costs: bool = False -) -> None: - build_dir = CURRENT_DIR / "build_cpp" - rmtree(build_dir, ignore_errors=True) - build_dir.mkdir(parents=True) - original_dir = Path.cwd() - chdir(build_dir) - - logger.info("Starting to compile the library.") - - cmake_cmd = ["cmake", str(CURRENT_DIR / "compile"), "-D__BUILD_FOR_PYTHON=ON"] - if integrated_opencl: - use_gpu = False - cmake_cmd.append("-D__INTEGRATE_OPENCL=ON") - if use_gpu: - cmake_cmd.append("-DUSE_GPU=ON") - if boost_root: - cmake_cmd.append(f"-DBOOST_ROOT={boost_root}") - if boost_dir: - cmake_cmd.append(f"-DBoost_DIR={boost_dir}") - if boost_include_dir: - cmake_cmd.append(f"-DBoost_INCLUDE_DIR={boost_include_dir}") - if boost_librarydir: - cmake_cmd.append(f"-DBOOST_LIBRARYDIR={boost_librarydir}") - if opencl_include_dir: - cmake_cmd.append(f"-DOpenCL_INCLUDE_DIR={opencl_include_dir}") - if opencl_library: - cmake_cmd.append(f"-DOpenCL_LIBRARY={opencl_library}") - elif use_cuda: - cmake_cmd.append("-DUSE_CUDA=ON") - if use_mpi: - cmake_cmd.append("-DUSE_MPI=ON") - if nomp: - cmake_cmd.append("-DUSE_OPENMP=OFF") - if use_hdfs: - cmake_cmd.append("-DUSE_HDFS=ON") - if time_costs: - cmake_cmd.append("-DUSE_TIMETAG=ON") - - if system() in {'Windows', 'Microsoft'}: - if use_mingw: - if use_mpi: - raise Exception('MPI version cannot be compiled by MinGW due to the miss of MPI library in it') - logger.info("Starting to compile with CMake and MinGW.") - # ref: https://stackoverflow.com/a/45104058/3986677 - silent_call(cmake_cmd + ["-G", "MinGW Makefiles", "-DCMAKE_SH=CMAKE_SH-NOTFOUND"], raise_error=True, - error_msg='Please install CMake and all required dependencies first') - silent_call(["mingw32-make.exe", "_lightgbm", f"-I{build_dir}", "-j4"], raise_error=True, - error_msg='Please install MinGW first') - else: - status = 1 - lib_path = CURRENT_DIR / "compile" / "windows" / "x64" / "DLL" / "lib_lightgbm.dll" - if not any((use_gpu, use_cuda, use_mpi, use_hdfs, nomp, bit32, integrated_opencl)): - logger.info("Starting to compile with MSBuild from existing solution file.") - platform_toolsets = ("v143", "v142", "v141", "v140") - for pt in platform_toolsets: - status = silent_call(["MSBuild", - str(CURRENT_DIR / "compile" / "windows" / "LightGBM.sln"), - "/p:Configuration=DLL", - "/p:Platform=x64", - f"/p:PlatformToolset={pt}"]) - if status == 0 and lib_path.is_file(): - break - else: - clear_path(CURRENT_DIR / "compile" / "windows" / "x64") - if status != 0 or not lib_path.is_file(): - logger.warning("Compilation with MSBuild from existing solution file failed.") - if status != 0 or not lib_path.is_file(): - arch = "Win32" if bit32 else "x64" - vs_versions = ( - "Visual Studio 17 2022", - "Visual Studio 16 2019", - "Visual Studio 15 2017", - "Visual Studio 14 2015" - ) - for vs in vs_versions: - logger.info(f"Starting to compile with {vs} ({arch}).") - status = silent_call(cmake_cmd + ["-G", vs, "-A", arch]) - if status == 0: - break - else: - clear_path(build_dir) - if status != 0: - raise Exception("\n".join(('Please install Visual Studio or MS Build and all required dependencies first', - LOG_NOTICE))) - silent_call(["cmake", "--build", str(build_dir), "--target", "_lightgbm", "--config", "Release"], raise_error=True, - error_msg='Please install CMake first') - else: # Linux, Darwin (macOS), etc. - logger.info("Starting to compile with CMake.") - silent_call(cmake_cmd, raise_error=True, error_msg='Please install CMake and all required dependencies first') - silent_call(["make", "_lightgbm", f"-I{build_dir}", "-j4"], raise_error=True, - error_msg='An error has occurred while building lightgbm library file') - chdir(original_dir) - - -class CustomInstallLib(install_lib): - - def install(self) -> List[str]: - outfiles = install_lib.install(self) - src = find_lib()[0] - dst = Path(self.install_dir) / 'lightgbm' - dst, _ = self.copy_file(src, str(dst)) - outfiles.append(dst) - return outfiles - - -class CustomInstall(install): - - user_options = install.user_options + LIGHTGBM_OPTIONS - - def initialize_options(self) -> None: - install.initialize_options(self) - self.mingw = False - self.integrated_opencl = False - self.gpu = False - self.cuda = False - self.boost_root = None - self.boost_dir = None - self.boost_include_dir = None - self.boost_librarydir = None - self.opencl_include_dir = None - self.opencl_library = None - self.mpi = False - self.hdfs = False - self.precompile = False - self.time_costs = False - self.nomp = False - self.bit32 = False - - def run(self) -> None: - if (8 * struct.calcsize("P")) != 64: - if self.bit32: - logger.warning("You're installing 32-bit version. " - "This version is slow and untested, so use it on your own risk.") - else: - raise Exception("Cannot install LightGBM in 32-bit Python, " - "please use 64-bit Python instead.") - LOG_PATH.touch() - if not self.precompile: - compile_cpp(use_mingw=self.mingw, use_gpu=self.gpu, use_cuda=self.cuda, use_mpi=self.mpi, - use_hdfs=self.hdfs, boost_root=self.boost_root, boost_dir=self.boost_dir, - boost_include_dir=self.boost_include_dir, boost_librarydir=self.boost_librarydir, - opencl_include_dir=self.opencl_include_dir, opencl_library=self.opencl_library, - nomp=self.nomp, bit32=self.bit32, integrated_opencl=self.integrated_opencl, - time_costs=self.time_costs) - install.run(self) - if LOG_PATH.is_file(): - LOG_PATH.unlink() - - -class CustomBdistWheel(bdist_wheel): - - user_options = bdist_wheel.user_options + LIGHTGBM_OPTIONS - - def initialize_options(self) -> None: - bdist_wheel.initialize_options(self) - self.mingw = False - self.integrated_opencl = False - self.gpu = False - self.cuda = False - self.boost_root = None - self.boost_dir = None - self.boost_include_dir = None - self.boost_librarydir = None - self.opencl_include_dir = None - self.opencl_library = None - self.mpi = False - self.hdfs = False - self.precompile = False - self.time_costs = False - self.nomp = False - self.bit32 = False - - def finalize_options(self) -> None: - bdist_wheel.finalize_options(self) - - install = self.reinitialize_command('install') - - install.mingw = self.mingw - install.integrated_opencl = self.integrated_opencl - install.gpu = self.gpu - install.cuda = self.cuda - install.boost_root = self.boost_root - install.boost_dir = self.boost_dir - install.boost_include_dir = self.boost_include_dir - install.boost_librarydir = self.boost_librarydir - install.opencl_include_dir = self.opencl_include_dir - install.opencl_library = self.opencl_library - install.mpi = self.mpi - install.hdfs = self.hdfs - install.precompile = self.precompile - install.time_costs = self.time_costs - install.nomp = self.nomp - install.bit32 = self.bit32 - - -class CustomSdist(sdist): - - def run(self) -> None: - IS_SOURCE_FLAG_PATH.touch() - rmtree(CURRENT_DIR / 'lightgbm' / 'Release', ignore_errors=True) - rmtree(CURRENT_DIR / 'lightgbm' / 'windows' / 'x64', ignore_errors=True) - lib_file = CURRENT_DIR / 'lightgbm' / 'lib_lightgbm.so' - if lib_file.is_file(): - lib_file.unlink() - sdist.run(self) - if IS_SOURCE_FLAG_PATH.is_file(): - IS_SOURCE_FLAG_PATH.unlink() - - -if __name__ == "__main__": - CURRENT_DIR = Path(__file__).absolute().parent - LOG_PATH = Path.home() / 'LightGBM_compilation.log' - LOG_NOTICE = f"The full version of error log was saved into {LOG_PATH}" - IS_SOURCE_FLAG_PATH = CURRENT_DIR / '_IS_SOURCE_PACKAGE.txt' - _version_file = CURRENT_DIR / 'lightgbm' / 'VERSION.txt' - version = _version_file.read_text(encoding='utf-8').strip() - readme = (CURRENT_DIR / 'README.rst').read_text(encoding='utf-8') - - sys.path.insert(0, str(CURRENT_DIR)) - - logging.basicConfig(level=logging.INFO) - logger = logging.getLogger('LightGBM') - - setup(name='lightgbm', - version=version, - description='LightGBM Python Package', - long_description=readme, - long_description_content_type='text/x-rst', - python_requires='>=3.6', - install_requires=[ - 'wheel', - 'numpy', - 'scipy', - 'scikit-learn!=0.22.0' - ], - extras_require={ - 'dask': [ - 'dask[array]>=2.0.0', - 'dask[dataframe]>=2.0.0', - 'dask[distributed]>=2.0.0', - 'pandas', - ], - }, - maintainer='Yu Shi', - maintainer_email='yushi2@microsoft.com', - zip_safe=False, - cmdclass={ - 'install': CustomInstall, - 'install_lib': CustomInstallLib, - 'bdist_wheel': CustomBdistWheel, - 'sdist': CustomSdist, - }, - packages=find_packages(), - include_package_data=True, - license='The MIT License (Microsoft)', - url='https://github.com/microsoft/LightGBM', - classifiers=['Development Status :: 5 - Production/Stable', - 'Intended Audience :: Science/Research', - 'License :: OSI Approved :: MIT License', - 'Natural Language :: English', - 'Operating System :: MacOS', - 'Operating System :: Microsoft :: Windows', - 'Operating System :: POSIX', - 'Operating System :: Unix', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Topic :: Scientific/Engineering :: Artificial Intelligence', - 'Typing :: Typed'])