[python-package] [ci] switch to PEP 517 / 518 builds (remove `setup.py`) (fixes #5061) (#5759)

.ci/test.sh

@@ -159,6 +159,9 @@ elif [[ $TASK == "bdist" ]]; then
         sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit -1
         mv \
             ./dist/*.whl \
+            ./dist/tmp.whl || exit -1
+        mv \
+            ./dist/tmp.whl \
             dist/lightgbm-$LGB_VER-py3-none-macosx_10_15_x86_64.macosx_11_6_x86_64.macosx_12_5_x86_64.whl || exit -1
         if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
             cp dist/lightgbm-$LGB_VER-py3-none-macosx*.whl $BUILD_ARTIFACTSTAGINGDIRECTORY || exit -1
@@ -173,6 +176,9 @@ elif [[ $TASK == "bdist" ]]; then
         cd $BUILD_DIRECTORY && sh ./build-python.sh bdist_wheel --integrated-opencl || exit -1
         mv \
             ./dist/*.whl \
+            ./dist/tmp.whl || exit -1
+        mv \
+            ./dist/tmp.whl \
             ./dist/lightgbm-$LGB_VER-py3-none-$PLATFORM.whl || exit -1
         sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit -1
         if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
@@ -186,12 +192,6 @@ elif [[ $TASK == "bdist" ]]; then
     exit 0
 fi
 
-# temporarily pin pip to versions that support 'pip install --install-option'
-# ref: https://github.com/microsoft/LightGBM/issues/5061#issuecomment-1510642287
-if [[ $METHOD == "pip" ]]; then
-    pip install 'pip<23.1'
-fi
-
 if [[ $TASK == "gpu" ]]; then
     sed -i'.bak' 's/std::string device_type = "cpu";/std::string device_type = "gpu";/' $BUILD_DIRECTORY/include/LightGBM/config.h
     grep -q 'std::string device_type = "gpu"' $BUILD_DIRECTORY/include/LightGBM/config.h || exit -1  # make sure that changes were really done
@@ -201,7 +201,7 @@ if [[ $TASK == "gpu" ]]; then
         pip install \
             --user \
             -v \
-            --install-option=--gpu \
+            --config-settings=cmake.define.USE_GPU=ON \
             $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER.tar.gz \
         || exit -1
         pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1
@@ -229,7 +229,7 @@ elif [[ $TASK == "cuda" ]]; then
         pip install \
             --user \
             -v \
-            --install-option=--cuda \
+            --config-settings=cmake.define.USE_CUDA=ON \
             $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER.tar.gz \
         || exit -1
         pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1
@@ -252,7 +252,7 @@ elif [[ $TASK == "mpi" ]]; then
         pip install \
             --user \
             -v \
-            --install-option=--mpi \
+            --config-settings=cmake.define.USE_MPI=ON \
             $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER.tar.gz \
         || exit -1
         pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1

.ci/test_windows.ps1

@@ -96,8 +96,8 @@ elseif ($env:TASK -eq "bdist") {
   cd $env:BUILD_SOURCESDIRECTORY
   sh "build-python.sh" bdist_wheel --integrated-opencl ; Check-Output $?
   sh $env:BUILD_SOURCESDIRECTORY/.ci/check_python_dists.sh $env:BUILD_SOURCESDIRECTORY/dist ; Check-Output $?
-  cd dist; pip install --user @(Get-ChildItem *.whl) ; Check-Output $?
+  cd dist; pip install --user @(Get-ChildItem *py3-none-win_amd64.whl) ; Check-Output $?
-  cp @(Get-ChildItem *.whl) $env:BUILD_ARTIFACTSTAGINGDIRECTORY
+  cp @(Get-ChildItem *py3-none-win_amd64.whl) $env:BUILD_ARTIFACTSTAGINGDIRECTORY
 } elseif (($env:APPVEYOR -eq "true") -and ($env:TASK -eq "python")) {
   cd $env:BUILD_SOURCESDIRECTORY
   if ($env:COMPILER -eq "MINGW") {

CMakeLists.txt

@@ -645,6 +645,10 @@ if(BUILD_CLI)
     )
 endif()
 
+if(__BUILD_FOR_PYTHON)
+    set(CMAKE_INSTALL_PREFIX "lightgbm")
+endif()
+
 install(
   TARGETS _lightgbm
   RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX}/bin

build-python.sh

@@ -94,75 +94,79 @@ while [ $# -gt 0 ]; do
             then shift;
         fi
         BOOST_DIR="${1#*=}"
-        BUILD_ARGS="${BUILD_ARGS} --boost-dir='${BOOST_DIR}'"
+        BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.Boost_DIR='${BOOST_DIR}'"
         ;;
     --boost-include-dir|--boost-include-dir=*)
         if [[ "$1" != *=* ]];
             then shift;
         fi
         BOOST_INCLUDE_DIR="${1#*=}"
-        BUILD_ARGS="${BUILD_ARGS} --boost-include-dir='${BOOST_INCLUDE_DIR}'"
+        BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.Boost_INCLUDE_DIR='${BOOST_INCLUDE_DIR}'"
         ;;
     --boost-librarydir|--boost-librarydir=*)
         if [[ "$1" != *=* ]];
             then shift;
         fi
         BOOST_LIBRARY_DIR="${1#*=}"
-        BUILD_ARGS="${BUILD_ARGS} --boost-librarydir='${BOOST_LIBRARY_DIR}'"
+        BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.BOOST_LIBRARYDIR='${BOOST_LIBRARY_DIR}'"
         ;;
     --boost-root|--boost-root=*)
         if [[ "$1" != *=* ]];
             then shift;
         fi
         BOOST_ROOT="${1#*=}"
-        BUILD_ARGS="${BUILD_ARGS} --boost-root='${BOOST_ROOT}'"
+        BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.Boost_ROOT='${BOOST_ROOT}'"
         ;;
     --opencl-include-dir|--opencl-include-dir=*)
         if [[ "$1" != *=* ]];
             then shift;
         fi
         OPENCL_INCLUDE_DIR="${1#*=}"
-        BUILD_ARGS="${BUILD_ARGS} --opencl-include-dir='${OPENCL_INCLUDE_DIR}'"
+        BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.OpenCL_INCLUDE_DIR='${OPENCL_INCLUDE_DIR}'"
         ;;
     --opencl-library|--opencl-library=*)
         if [[ "$1" != *=* ]];
             then shift;
         fi
         OPENCL_LIBRARY="${1#*=}"
-        BUILD_ARGS="${BUILD_ARGS} --opencl-library='${OPENCL_LIBRARY}'"
+        BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.OpenCL_LIBRARY='${OPENCL_LIBRARY}'"
         ;;
     #########
     # flags #
     #########
     --bit32)
-        BUILD_ARGS="${BUILD_ARGS} --bit32"
+        export CMAKE_GENERATOR="Visual Studio 17 2022"
+        export CMAKE_GENERATOR_PLATFORM="Win32"
+        echo "[INFO] Attempting to build 32-bit version of LightGBM, which is only supported on Windows with generator '${CMAKE_GENERATOR}'."
         ;;
     --cuda)
-        BUILD_ARGS="${BUILD_ARGS} --cuda"
+        BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.USE_CUDA=ON"
         ;;
     --gpu)
-        BUILD_ARGS="${BUILD_ARGS} --gpu"
+        BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.USE_GPU=ON"
         ;;
     --hdfs)
-        BUILD_ARGS="${BUILD_ARGS} --hdfs"
+        BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.USE_HDFS=ON"
         ;;
     --integrated-opencl)
-        BUILD_ARGS="${BUILD_ARGS} --integrated-opencl"
+        BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.__INTEGRATE_OPENCL=ON"
         ;;
     --mingw)
-        BUILD_ARGS="${BUILD_ARGS} --mingw"
+        export CMAKE_GENERATOR='MinGW Makefiles'
+        # ref: https://stackoverflow.com/a/45104058/3986677
+        BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.CMAKE_SH=CMAKE_SH-NOTFOUND"
         ;;
     --mpi)
-        BUILD_ARGS="${BUILD_ARGS} --mpi"
+        BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.USE_MPI=ON"
         ;;
     --nomp)
-        BUILD_ARGS="${BUILD_ARGS} --nomp"
+        BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.USE_OPENMP=OFF"
         ;;
     --precompile)
         PRECOMPILE="true"
         ;;
     --time-costs)
-        BUILD_ARGS="${PIP_INSTALL_ARGS} --time-costs"
+        BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.USE_TIMETAG=ON"
         ;;
     --user)
         PIP_INSTALL_ARGS="${PIP_INSTALL_ARGS} --user"
@@ -175,6 +179,8 @@ while [ $# -gt 0 ]; do
   shift
 done
 
+pip install --prefer-binary 'build>=0.10.0'
+
 # create a new directory that just contains the files needed
 # to build the Python package
 create_isolated_source_dir() {
@@ -189,21 +195,15 @@ create_isolated_source_dir() {
 
     cp -R ./python-package ./lightgbm-python
 
-    # temporarily remove these files until
-    # https://github.com/microsoft/LightGBM/issues/5061 is done
-    rm ./lightgbm-python/pyproject.toml
-    rm ./lightgbm-python/setup.cfg
-
     cp LICENSE ./lightgbm-python/
     cp VERSION.txt ./lightgbm-python/lightgbm/VERSION.txt
 
-    mkdir -p ./lightgbm-python/compile
+    cp -R ./cmake ./lightgbm-python
-    cp -R ./cmake ./lightgbm-python/compile
+    cp CMakeLists.txt ./lightgbm-python
-    cp CMakeLists.txt ./lightgbm-python/compile
+    cp -R ./include ./lightgbm-python
-    cp -R ./include ./lightgbm-python/compile
+    cp -R ./src ./lightgbm-python
-    cp -R ./src ./lightgbm-python/compile
+    cp -R ./swig ./lightgbm-python
-    cp -R ./swig ./lightgbm-python/compile
+    cp -R ./windows ./lightgbm-python
-    cp -R ./windows ./lightgbm-python/compile
 
     # include only specific files from external_libs, to keep the package
     # small and avoid redistributing code with licenses incompatible with
@@ -212,77 +212,77 @@ create_isolated_source_dir() {
     ######################
     # fast_double_parser #
     ######################
-    mkdir -p ./lightgbm-python/compile/external_libs/fast_double_parser
+    mkdir -p ./lightgbm-python/external_libs/fast_double_parser
     cp \
         external_libs/fast_double_parser/CMakeLists.txt \
-        ./lightgbm-python/compile/external_libs/fast_double_parser/CMakeLists.txt
+        ./lightgbm-python/external_libs/fast_double_parser/CMakeLists.txt
     cp \
         external_libs/fast_double_parser/LICENSE* \
-        ./lightgbm-python/compile/external_libs/fast_double_parser/
+        ./lightgbm-python/external_libs/fast_double_parser/
 
-    mkdir -p ./lightgbm-python/compile/external_libs/fast_double_parser/include/
+    mkdir -p ./lightgbm-python/external_libs/fast_double_parser/include/
     cp \
         external_libs/fast_double_parser/include/fast_double_parser.h \
-        ./lightgbm-python/compile/external_libs/fast_double_parser/include/
+        ./lightgbm-python/external_libs/fast_double_parser/include/
 
     #######
     # fmt #
     #######
-    mkdir -p ./lightgbm-python/compile/external_libs/fmt
+    mkdir -p ./lightgbm-python/external_libs/fmt
     cp \
         external_libs/fast_double_parser/CMakeLists.txt \
-        ./lightgbm-python/compile/external_libs/fmt/CMakeLists.txt
+        ./lightgbm-python/external_libs/fmt/CMakeLists.txt
     cp \
         external_libs/fmt/LICENSE* \
-        ./lightgbm-python/compile/external_libs/fmt/
+        ./lightgbm-python/external_libs/fmt/
 
-    mkdir -p ./lightgbm-python/compile/external_libs/fmt/include/fmt
+    mkdir -p ./lightgbm-python/external_libs/fmt/include/fmt
     cp \
         external_libs/fmt/include/fmt/*.h \
-        ./lightgbm-python/compile/external_libs/fmt/include/fmt/
+        ./lightgbm-python/external_libs/fmt/include/fmt/
 
     #########
     # Eigen #
     #########
-    mkdir -p ./lightgbm-python/compile/external_libs/eigen/Eigen
+    mkdir -p ./lightgbm-python/external_libs/eigen/Eigen
     cp \
         external_libs/eigen/CMakeLists.txt \
-        ./lightgbm-python/compile/external_libs/eigen/CMakeLists.txt
+        ./lightgbm-python/external_libs/eigen/CMakeLists.txt
 
     modules="Cholesky Core Dense Eigenvalues Geometry Householder Jacobi LU QR SVD"
     for eigen_module in ${modules}; do
         cp \
             external_libs/eigen/Eigen/${eigen_module} \
-            ./lightgbm-python/compile/external_libs/eigen/Eigen/${eigen_module}
+            ./lightgbm-python/external_libs/eigen/Eigen/${eigen_module}
         if [ ${eigen_module} != "Dense" ]; then
-            mkdir -p ./lightgbm-python/compile/external_libs/eigen/Eigen/src/${eigen_module}/
+            mkdir -p ./lightgbm-python/external_libs/eigen/Eigen/src/${eigen_module}/
             cp \
                 -R \
                 external_libs/eigen/Eigen/src/${eigen_module}/* \
-                ./lightgbm-python/compile/external_libs/eigen/Eigen/src/${eigen_module}/
+                ./lightgbm-python/external_libs/eigen/Eigen/src/${eigen_module}/
         fi
     done
 
-    mkdir -p ./lightgbm-python/compile/external_libs/eigen/Eigen/misc
+    mkdir -p ./lightgbm-python/external_libs/eigen/Eigen/misc
     cp \
         -R \
         external_libs/eigen/Eigen/src/misc \
-        ./lightgbm-python/compile/external_libs/eigen/Eigen/src/misc/
+        ./lightgbm-python/external_libs/eigen/Eigen/src/misc/
 
-    mkdir -p ./lightgbm-python/compile/external_libs/eigen/Eigen/plugins
+    mkdir -p ./lightgbm-python/external_libs/eigen/Eigen/plugins
     cp \
         -R \
         external_libs/eigen/Eigen/src/plugins \
-        ./lightgbm-python/compile/external_libs/eigen/Eigen/src/plugins/
+        ./lightgbm-python/external_libs/eigen/Eigen/src/plugins/
 
     ###################
     # compute (Boost) #
     ###################
-    mkdir -p ./lightgbm-python/compile/external_libs/compute
+    mkdir -p ./lightgbm-python/external_libs/compute
     cp \
         -R \
         external_libs/compute/include \
-        ./lightgbm-python/compile/external_libs/compute/include/
+        ./lightgbm-python/external_libs/compute/include/
 }
 
 create_isolated_source_dir
@@ -292,9 +292,39 @@ cd ./lightgbm-python
 # installation involves building the wheel + `pip install`-ing it
 if test "${INSTALL}" = true; then
     if test "${PRECOMPILE}" = true; then
-        echo "--- installing lightgbm (from precompiled lib_lightgbm) ---"
+        BUILD_SDIST=true
-        python setup.py install ${PIP_INSTALL_ARGS} --precompile
+        BUILD_WHEEL=false
-        exit 0
+        BUILD_ARGS=""
+        rm -rf \
+            ./cmake \
+            ./CMakeLists.txt \
+            ./external_libs \
+            ./include \
+            ./src \
+            ./swig \
+            ./windows
+        # use regular-old setuptools for these builds, to avoid
+        # trying to recompile the shared library
+        sed -i.bak -e '/start:build-system/,/end:build-system/d' pyproject.toml
+        echo '[build-system]' >> ./pyproject.toml
+        echo 'requires = ["setuptools"]' >> ./pyproject.toml
+        echo 'build-backend = "setuptools.build_meta"' >> ./pyproject.toml
+        echo "" >> ./pyproject.toml
+        echo "recursive-include lightgbm *.dll *.so" > ./MANIFEST.in
+        echo "" >> ./MANIFEST.in
+        mkdir -p ./lightgbm/lib
+        if test -f ../lib_lightgbm.so; then
+            echo "found pre-compiled lib_lightgbm.so"
+            cp ../lib_lightgbm.so ./lightgbm/lib/lib_lightgbm.so
+        elif test -f ../Release/lib_lightgbm.dll; then
+            echo "found pre-compiled Release/lib_lightgbm.dll"
+            cp ../Release/lib_lightgbm.dll ./lightgbm/lib/lib_lightgbm.dll
+        elif test -f ../windows/x64/DLL/lib_lightgbm.dll; then
+            echo "found pre-compiled windows/x64/DLL/lib_lightgbm.dll"
+            cp ../windows/x64/DLL/lib_lightgbm.dll ./lightgbm/lib/lib_lightgbm.dll
+            cp ../windows/x64/DLL/lib_lightgbm.lib ./lightgbm/lib/lib_lightgbm.lib
+        fi
+        rm -f ./*.bak
     else
         BUILD_SDIST="false"
         BUILD_WHEEL="true"
@@ -304,16 +334,20 @@ fi
 if test "${BUILD_SDIST}" = true; then
     echo "--- building sdist ---"
     rm -f ../dist/*.tar.gz
-    python ./setup.py sdist \
+    python -m build \
-        --dist-dir ../dist
+        --sdist \
+        --outdir ../dist \
+        .
 fi
 
 if test "${BUILD_WHEEL}" = true; then
     echo "--- building wheel ---"
     rm -f ../dist/*.whl || true
-    python setup.py bdist_wheel \
+    python -m build \
-        --dist-dir ../dist \
+        --wheel \
-        ${BUILD_ARGS}
+        --outdir ../dist \
+        ${BUILD_ARGS} \
+        .
 fi
 
 if test "${INSTALL}" = true; then

python-package/MANIFEST.in

@@ -1,47 +0,0 @@
-prune build
-include LICENSE
-include *.rst *.txt
-recursive-include lightgbm VERSION.txt py.typed *.py *.so
-include compile/CMakeLists.txt
-include compile/cmake/IntegratedOpenCL.cmake
-recursive-include compile *.so
-recursive-include compile/Release *.dll
-include compile/external_libs/compute/CMakeLists.txt
-recursive-include compile/external_libs/compute/cmake *
-recursive-include compile/external_libs/compute/include *
-recursive-include compile/external_libs/compute/meta *
-include compile/external_libs/eigen/CMakeLists.txt
-include compile/external_libs/eigen/Eigen/Cholesky
-include compile/external_libs/eigen/Eigen/Core
-include compile/external_libs/eigen/Eigen/Dense
-include compile/external_libs/eigen/Eigen/Eigenvalues
-include compile/external_libs/eigen/Eigen/Geometry
-include compile/external_libs/eigen/Eigen/Householder
-include compile/external_libs/eigen/Eigen/Jacobi
-include compile/external_libs/eigen/Eigen/LU
-include compile/external_libs/eigen/Eigen/QR
-include compile/external_libs/eigen/Eigen/SVD
-recursive-include compile/external_libs/eigen/Eigen/src/Cholesky *
-recursive-include compile/external_libs/eigen/Eigen/src/Core *
-recursive-include compile/external_libs/eigen/Eigen/src/Eigenvalues *
-recursive-include compile/external_libs/eigen/Eigen/src/Geometry *
-recursive-include compile/external_libs/eigen/Eigen/src/Householder *
-recursive-include compile/external_libs/eigen/Eigen/src/Jacobi *
-recursive-include compile/external_libs/eigen/Eigen/src/LU *
-recursive-include compile/external_libs/eigen/Eigen/src/misc *
-recursive-include compile/external_libs/eigen/Eigen/src/plugins *
-recursive-include compile/external_libs/eigen/Eigen/src/QR *
-recursive-include compile/external_libs/eigen/Eigen/src/SVD *
-include compile/external_libs/fast_double_parser/CMakeLists.txt
-include compile/external_libs/fast_double_parser/LICENSE
-include compile/external_libs/fast_double_parser/LICENSE.BSL
-recursive-include compile/external_libs/fast_double_parser/include *
-include compile/external_libs/fmt/CMakeLists.txt
-include compile/external_libs/fmt/LICENSE.rst
-recursive-include compile/external_libs/fmt/include *
-recursive-include compile/include *
-recursive-include compile/src *
-recursive-include compile/windows LightGBM.sln LightGBM.vcxproj
-recursive-include compile/windows/x64/DLL *.dll
-global-exclude *.py[co]
-exclude compile/external_libs/compute/.git

python-package/README.rst

@@ -60,7 +60,7 @@ Build Threadless Version
 
 .. code:: sh
 
-    pip install lightgbm --install-option=--nomp
+    pip install lightgbm --config-settings=cmake.define.USE_OPENMP=OFF
 
 All requirements, except the **OpenMP** requirement, from `Build from Sources section <#build-from-sources>`__ apply for this installation option as well.
 
@@ -71,7 +71,7 @@ Build MPI Version
 
 .. code:: sh
 
-    pip install lightgbm --install-option=--mpi
+    pip install lightgbm --config-settings=cmake.define.USE_MPI=ON
 
 All requirements from `Build from Sources section <#build-from-sources>`__ apply for this installation option as well.
 
@@ -84,7 +84,7 @@ Build GPU Version
 
 .. code:: sh
 
-    pip install lightgbm --install-option=--gpu
+    pip install lightgbm --config-settings=cmake.define.USE_GPU=ON
 
 All requirements from `Build from Sources section <#build-from-sources>`__ apply for this installation option as well.
 
@@ -94,21 +94,24 @@ For **Windows** users, `CMake`_ (version 3.8 or higher) is strongly required.
 
 .. code:: sh
 
-    pip install lightgbm --install-option=--gpu --install-option="--opencl-include-dir=/usr/local/cuda/include/" --install-option="--opencl-library=/usr/local/cuda/lib64/libOpenCL.so"
+    pip install lightgbm \
+      --config-settings=cmake.define.USE_GPU=ON \
+      --config-settings=cmake.define.OpenCL_INCLUDE_DIR="/usr/local/cuda/include/" \
+      --config-settings=cmake.define.OpenCL_LIBRARY="/usr/local/cuda/lib64/libOpenCL.so"
 
-All available options:
+All available options that can be passed via ``cmake.define.{option}``.
 
-- boost-root
+- Boost_ROOT
 
-- boost-dir
+- Boost_DIR
 
-- boost-include-dir
+- Boost_INCLUDE_DIR
 
-- boost-librarydir
+- BOOST_LIBRARYDIR
 
-- opencl-include-dir
+- OpenCL_INCLUDE_DIR
 
-- opencl-library
+- OpenCL_LIBRARY
 
 For more details see `FindBoost <https://cmake.org/cmake/help/latest/module/FindBoost.html>`__ and `FindOpenCL <https://cmake.org/cmake/help/latest/module/FindOpenCL.html>`__.
 
@@ -117,7 +120,7 @@ Build CUDA Version
 
 .. code:: sh
 
-    pip install lightgbm --install-option=--cuda
+    pip install lightgbm --config-settings=cmake.define.USE_CUDA=ON
 
 All requirements from `Build from Sources section <#build-from-sources>`__ apply for this installation option as well, and `CMake`_ (version 3.16 or higher) is strongly required.
 
@@ -130,7 +133,7 @@ Build HDFS Version
 
 .. code:: sh
 
-    pip install lightgbm --install-option=--hdfs
+    pip install lightgbm --config-settings=cmake.define.USE_HDFS=ON
 
 All requirements from `Build from Sources section <#build-from-sources>`__ apply for this installation option as well.
 
@@ -143,7 +146,9 @@ Build with MinGW-w64 on Windows
 
 .. code:: sh
 
-    pip install lightgbm --install-option=--mingw
+    # in sh.exe, git bash, or other Unix-like shell
+    export CMAKE_GENERATOR='MinGW Makefiles'
+    pip install lightgbm --config-settings=cmake.define.CMAKE_SH=CMAKE_SH-NOTFOUND
 
 `CMake`_ and `MinGW-w64 <https://www.mingw-w64.org/>`_ should be installed first.
 
@@ -155,7 +160,10 @@ Build 32-bit Version with 32-bit Python
 
 .. code:: sh
 
-    pip install lightgbm --install-option=--bit32
+    # in sh.exe, git bash, or other Unix-like shell
+    export CMAKE_GENERATOR='Visual Studio 17 2022'
+    export CMAKE_GENERATOR_PLATFORM='Win32'
+    pip install --no-binary lightgbm lightgbm
 
 By default, installation in environment with 32-bit Python is prohibited. However, you can remove this prohibition on your own risk by passing ``bit32`` option.
 
@@ -166,7 +174,7 @@ Build with Time Costs Output
 
 .. code:: sh
 
-    pip install lightgbm --install-option=--time-costs
+    pip install lightgbm --config-settings=cmake.define.USE_TIMETAG=ON
 
 Use this option to make LightGBM output time costs for different internal routines, to investigate and benchmark its performance.
 
@@ -221,6 +229,21 @@ Build Wheel File
 
 You can use ``sh ./build-python.sh install bdist_wheel`` instead of ``sh ./build-python.sh install`` to build wheel file and use it for installation later. This might be useful for systems with restricted or completely without network access.
 
+Build With MSBuild
+******************
+
+To use ``MSBuild`` (Windows-only), first build ``lib_lightgbm.dll`` by running the following from the root of the repo.
+
+.. code:: sh
+
+  MSBuild.exe windows/LightGBM.sln /p:Configuration=DLL /p:Platform=x64 /p:PlatformToolset=v143
+
+Then install the Python package using that library.
+
+.. code:: sh
+
+  sh ./build-python.sh install --precompile
+
 Install Dask-package
 ''''''''''''''''''''
 

python-package/lightgbm/libpath.py

@@ -15,15 +15,12 @@ def find_lib_path() -> List[str]:
     lib_path: list of str
        List of all found library paths to LightGBM.
     """
-    curr_path = Path(__file__).absolute().parent
+    curr_path = Path(__file__).absolute()
     dll_path = [curr_path,
                 curr_path.parents[1],
-                curr_path / 'compile',
+                curr_path.parents[0] / 'bin',
-                curr_path.parent / 'compile',
+                curr_path.parents[0] / 'lib']
-                curr_path.parents[1] / 'lib']
     if system() in ('Windows', 'Microsoft'):
-        dll_path.append(curr_path.parent / 'compile' / 'Release')
-        dll_path.append(curr_path.parent / 'compile' / 'windows' / 'x64' / 'DLL')
         dll_path.append(curr_path.parents[1] / 'Release')
         dll_path.append(curr_path.parents[1] / 'windows' / 'x64' / 'DLL')
         dll_path = [p / 'lib_lightgbm.dll' for p in dll_path]

python-package/pyproject.toml

@@ -1,3 +1,63 @@
+[project]
+
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Natural Language :: English",
+    "Operating System :: MacOS",
+    "Operating System :: Microsoft :: Windows",
+    "Operating System :: POSIX",
+    "Operating System :: Unix",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.7",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence"
+]
+description = "LightGBM Python Package"
+license = {file = "LICENSE"}
+maintainers = [
+  {name = "Yu Shi", email = "yushi@microsoft.com"}
+]
+name = "lightgbm"
+readme = "README.rst"
+requires-python = ">=3.6"
+version = "3.3.5.99"
+
+[project.urls]
+homepage = "https://github.com/microsoft/LightGBM"
+documentation = "https://lightgbm.readthedocs.io/en/latest/"
+repository = "https://github.com/microsoft/LightGBM.git"
+changelog = "https://github.com/microsoft/LightGBM/releases"
+
+# start:build-system
+[build-system]
+
+requires = ["scikit-build-core>=0.4.4"]
+build-backend = "scikit_build_core.build"
+
+# based on https://github.com/scikit-build/scikit-build-core#configuration
+[tool.scikit-build]
+
+cmake.minimum-version = "3.15"
+ninja.minimum-version = "1.11"
+ninja.make-fallback = true
+cmake.args = [
+    "-D__BUILD_FOR_PYTHON:BOOL=ON"
+]
+cmake.verbose = false
+cmake.build-type = "Release"
+logging.level = "INFO"
+sdist.reproducible = true
+wheel.py-api = "py3"
+experimental = false
+strict-config = true
+minimum-version = "0.4.4"
+
+# end:build-system
+
 [tool.isort]
 line_length = 120
 skip_glob = [

python-package/setup.cfg

@@ -1,3 +1,20 @@
+[options]
+include_package_data = True
+install_requires =
+    numpy
+    scikit-learn!=0.22.0
+    scipy
+
+[options.extras_require]
+dask =
+    dask[array]>=2.0.0
+    dask[dataframe]>=2.0.0
+    dask[distributed]>=2.0.0
+    pandas
+
+[options.packages.find]
+where = lightgbm
+
 [flake8]
 ignore =
     # line too long

python-package/setup.py

@@ -1,355 +0,0 @@
-# coding: utf-8
-"""Setup lightgbm package."""
-import logging
-import struct
-import subprocess
-import sys
-from os import chdir
-from pathlib import Path
-from platform import system
-from shutil import rmtree
-from typing import List, Optional
-
-from setuptools import find_packages, setup
-from setuptools.command.install import install
-from setuptools.command.install_lib import install_lib
-from setuptools.command.sdist import sdist
-from wheel.bdist_wheel import bdist_wheel
-
-LIGHTGBM_OPTIONS = [
-    ('mingw', 'm', 'Compile with MinGW'),
-    ('integrated-opencl', None, 'Compile integrated OpenCL version'),
-    ('gpu', 'g', 'Compile GPU version'),
-    ('cuda', None, 'Compile CUDA version'),
-    ('mpi', None, 'Compile MPI version'),
-    ('nomp', None, 'Compile version without OpenMP support'),
-    ('hdfs', 'h', 'Compile HDFS version'),
-    ('bit32', None, 'Compile 32-bit version'),
-    ('precompile', 'p', 'Use precompiled library'),
-    ('time-costs', None, 'Output time costs for different internal routines'),
-    ('boost-root=', None, 'Boost preferred installation prefix'),
-    ('boost-dir=', None, 'Directory with Boost package configuration file'),
-    ('boost-include-dir=', None, 'Directory containing Boost headers'),
-    ('boost-librarydir=', None, 'Preferred Boost library directory'),
-    ('opencl-include-dir=', None, 'OpenCL include directory'),
-    ('opencl-library=', None, 'Path to OpenCL library')
-]
-
-
-def find_lib() -> List[str]:
-    libpath_py = CURRENT_DIR / 'lightgbm' / 'libpath.py'
-    libpath = {'__file__': libpath_py}
-    exec(compile(libpath_py.read_bytes(), libpath_py, 'exec'), libpath, libpath)
-
-    LIB_PATH = libpath['find_lib_path']()  # type: ignore
-    logger.info(f"Installing lib_lightgbm from: {LIB_PATH}")
-    return LIB_PATH
-
-
-def clear_path(path: Path) -> None:
-    if path.is_dir():
-        for file_name in path.iterdir():
-            if file_name.is_dir():
-                rmtree(file_name)
-            else:
-                file_name.unlink()
-
-
-def silent_call(cmd: List[str], raise_error: bool = False, error_msg: str = '') -> int:
-    try:
-        with open(LOG_PATH, "ab") as log:
-            subprocess.check_call(cmd, stderr=log, stdout=log)
-        return 0
-    except Exception:
-        if raise_error:
-            raise Exception("\n".join((error_msg, LOG_NOTICE)))
-        return 1
-
-
-def compile_cpp(
-    use_mingw: bool = False,
-    use_gpu: bool = False,
-    use_cuda: bool = False,
-    use_mpi: bool = False,
-    use_hdfs: bool = False,
-    boost_root: Optional[str] = None,
-    boost_dir: Optional[str] = None,
-    boost_include_dir: Optional[str] = None,
-    boost_librarydir: Optional[str] = None,
-    opencl_include_dir: Optional[str] = None,
-    opencl_library: Optional[str] = None,
-    nomp: bool = False,
-    bit32: bool = False,
-    integrated_opencl: bool = False,
-    time_costs: bool = False
-) -> None:
-    build_dir = CURRENT_DIR / "build_cpp"
-    rmtree(build_dir, ignore_errors=True)
-    build_dir.mkdir(parents=True)
-    original_dir = Path.cwd()
-    chdir(build_dir)
-
-    logger.info("Starting to compile the library.")
-
-    cmake_cmd = ["cmake", str(CURRENT_DIR / "compile"), "-D__BUILD_FOR_PYTHON=ON"]
-    if integrated_opencl:
-        use_gpu = False
-        cmake_cmd.append("-D__INTEGRATE_OPENCL=ON")
-    if use_gpu:
-        cmake_cmd.append("-DUSE_GPU=ON")
-        if boost_root:
-            cmake_cmd.append(f"-DBOOST_ROOT={boost_root}")
-        if boost_dir:
-            cmake_cmd.append(f"-DBoost_DIR={boost_dir}")
-        if boost_include_dir:
-            cmake_cmd.append(f"-DBoost_INCLUDE_DIR={boost_include_dir}")
-        if boost_librarydir:
-            cmake_cmd.append(f"-DBOOST_LIBRARYDIR={boost_librarydir}")
-        if opencl_include_dir:
-            cmake_cmd.append(f"-DOpenCL_INCLUDE_DIR={opencl_include_dir}")
-        if opencl_library:
-            cmake_cmd.append(f"-DOpenCL_LIBRARY={opencl_library}")
-    elif use_cuda:
-        cmake_cmd.append("-DUSE_CUDA=ON")
-    if use_mpi:
-        cmake_cmd.append("-DUSE_MPI=ON")
-    if nomp:
-        cmake_cmd.append("-DUSE_OPENMP=OFF")
-    if use_hdfs:
-        cmake_cmd.append("-DUSE_HDFS=ON")
-    if time_costs:
-        cmake_cmd.append("-DUSE_TIMETAG=ON")
-
-    if system() in {'Windows', 'Microsoft'}:
-        if use_mingw:
-            if use_mpi:
-                raise Exception('MPI version cannot be compiled by MinGW due to the miss of MPI library in it')
-            logger.info("Starting to compile with CMake and MinGW.")
-            # ref: https://stackoverflow.com/a/45104058/3986677
-            silent_call(cmake_cmd + ["-G", "MinGW Makefiles", "-DCMAKE_SH=CMAKE_SH-NOTFOUND"], raise_error=True,
-                        error_msg='Please install CMake and all required dependencies first')
-            silent_call(["mingw32-make.exe", "_lightgbm", f"-I{build_dir}", "-j4"], raise_error=True,
-                        error_msg='Please install MinGW first')
-        else:
-            status = 1
-            lib_path = CURRENT_DIR / "compile" / "windows" / "x64" / "DLL" / "lib_lightgbm.dll"
-            if not any((use_gpu, use_cuda, use_mpi, use_hdfs, nomp, bit32, integrated_opencl)):
-                logger.info("Starting to compile with MSBuild from existing solution file.")
-                platform_toolsets = ("v143", "v142", "v141", "v140")
-                for pt in platform_toolsets:
-                    status = silent_call(["MSBuild",
-                                          str(CURRENT_DIR / "compile" / "windows" / "LightGBM.sln"),
-                                          "/p:Configuration=DLL",
-                                          "/p:Platform=x64",
-                                          f"/p:PlatformToolset={pt}"])
-                    if status == 0 and lib_path.is_file():
-                        break
-                    else:
-                        clear_path(CURRENT_DIR / "compile" / "windows" / "x64")
-                if status != 0 or not lib_path.is_file():
-                    logger.warning("Compilation with MSBuild from existing solution file failed.")
-            if status != 0 or not lib_path.is_file():
-                arch = "Win32" if bit32 else "x64"
-                vs_versions = (
-                    "Visual Studio 17 2022",
-                    "Visual Studio 16 2019",
-                    "Visual Studio 15 2017",
-                    "Visual Studio 14 2015"
-                )
-                for vs in vs_versions:
-                    logger.info(f"Starting to compile with {vs} ({arch}).")
-                    status = silent_call(cmake_cmd + ["-G", vs, "-A", arch])
-                    if status == 0:
-                        break
-                    else:
-                        clear_path(build_dir)
-                if status != 0:
-                    raise Exception("\n".join(('Please install Visual Studio or MS Build and all required dependencies first',
-                                    LOG_NOTICE)))
-                silent_call(["cmake", "--build", str(build_dir), "--target", "_lightgbm", "--config", "Release"], raise_error=True,
-                            error_msg='Please install CMake first')
-    else:  # Linux, Darwin (macOS), etc.
-        logger.info("Starting to compile with CMake.")
-        silent_call(cmake_cmd, raise_error=True, error_msg='Please install CMake and all required dependencies first')
-        silent_call(["make", "_lightgbm", f"-I{build_dir}", "-j4"], raise_error=True,
-                    error_msg='An error has occurred while building lightgbm library file')
-    chdir(original_dir)
-
-
-class CustomInstallLib(install_lib):
-
-    def install(self) -> List[str]:
-        outfiles = install_lib.install(self)
-        src = find_lib()[0]
-        dst = Path(self.install_dir) / 'lightgbm'
-        dst, _ = self.copy_file(src, str(dst))
-        outfiles.append(dst)
-        return outfiles
-
-
-class CustomInstall(install):
-
-    user_options = install.user_options + LIGHTGBM_OPTIONS
-
-    def initialize_options(self) -> None:
-        install.initialize_options(self)
-        self.mingw = False
-        self.integrated_opencl = False
-        self.gpu = False
-        self.cuda = False
-        self.boost_root = None
-        self.boost_dir = None
-        self.boost_include_dir = None
-        self.boost_librarydir = None
-        self.opencl_include_dir = None
-        self.opencl_library = None
-        self.mpi = False
-        self.hdfs = False
-        self.precompile = False
-        self.time_costs = False
-        self.nomp = False
-        self.bit32 = False
-
-    def run(self) -> None:
-        if (8 * struct.calcsize("P")) != 64:
-            if self.bit32:
-                logger.warning("You're installing 32-bit version. "
-                               "This version is slow and untested, so use it on your own risk.")
-            else:
-                raise Exception("Cannot install LightGBM in 32-bit Python, "
-                                "please use 64-bit Python instead.")
-        LOG_PATH.touch()
-        if not self.precompile:
-            compile_cpp(use_mingw=self.mingw, use_gpu=self.gpu, use_cuda=self.cuda, use_mpi=self.mpi,
-                        use_hdfs=self.hdfs, boost_root=self.boost_root, boost_dir=self.boost_dir,
-                        boost_include_dir=self.boost_include_dir, boost_librarydir=self.boost_librarydir,
-                        opencl_include_dir=self.opencl_include_dir, opencl_library=self.opencl_library,
-                        nomp=self.nomp, bit32=self.bit32, integrated_opencl=self.integrated_opencl,
-                        time_costs=self.time_costs)
-        install.run(self)
-        if LOG_PATH.is_file():
-            LOG_PATH.unlink()
-
-
-class CustomBdistWheel(bdist_wheel):
-
-    user_options = bdist_wheel.user_options + LIGHTGBM_OPTIONS
-
-    def initialize_options(self) -> None:
-        bdist_wheel.initialize_options(self)
-        self.mingw = False
-        self.integrated_opencl = False
-        self.gpu = False
-        self.cuda = False
-        self.boost_root = None
-        self.boost_dir = None
-        self.boost_include_dir = None
-        self.boost_librarydir = None
-        self.opencl_include_dir = None
-        self.opencl_library = None
-        self.mpi = False
-        self.hdfs = False
-        self.precompile = False
-        self.time_costs = False
-        self.nomp = False
-        self.bit32 = False
-
-    def finalize_options(self) -> None:
-        bdist_wheel.finalize_options(self)
-
-        install = self.reinitialize_command('install')
-
-        install.mingw = self.mingw
-        install.integrated_opencl = self.integrated_opencl
-        install.gpu = self.gpu
-        install.cuda = self.cuda
-        install.boost_root = self.boost_root
-        install.boost_dir = self.boost_dir
-        install.boost_include_dir = self.boost_include_dir
-        install.boost_librarydir = self.boost_librarydir
-        install.opencl_include_dir = self.opencl_include_dir
-        install.opencl_library = self.opencl_library
-        install.mpi = self.mpi
-        install.hdfs = self.hdfs
-        install.precompile = self.precompile
-        install.time_costs = self.time_costs
-        install.nomp = self.nomp
-        install.bit32 = self.bit32
-
-
-class CustomSdist(sdist):
-
-    def run(self) -> None:
-        IS_SOURCE_FLAG_PATH.touch()
-        rmtree(CURRENT_DIR / 'lightgbm' / 'Release', ignore_errors=True)
-        rmtree(CURRENT_DIR / 'lightgbm' / 'windows' / 'x64', ignore_errors=True)
-        lib_file = CURRENT_DIR / 'lightgbm' / 'lib_lightgbm.so'
-        if lib_file.is_file():
-            lib_file.unlink()
-        sdist.run(self)
-        if IS_SOURCE_FLAG_PATH.is_file():
-            IS_SOURCE_FLAG_PATH.unlink()
-
-
-if __name__ == "__main__":
-    CURRENT_DIR = Path(__file__).absolute().parent
-    LOG_PATH = Path.home() / 'LightGBM_compilation.log'
-    LOG_NOTICE = f"The full version of error log was saved into {LOG_PATH}"
-    IS_SOURCE_FLAG_PATH = CURRENT_DIR / '_IS_SOURCE_PACKAGE.txt'
-    _version_file = CURRENT_DIR / 'lightgbm' / 'VERSION.txt'
-    version = _version_file.read_text(encoding='utf-8').strip()
-    readme = (CURRENT_DIR / 'README.rst').read_text(encoding='utf-8')
-
-    sys.path.insert(0, str(CURRENT_DIR))
-
-    logging.basicConfig(level=logging.INFO)
-    logger = logging.getLogger('LightGBM')
-
-    setup(name='lightgbm',
-          version=version,
-          description='LightGBM Python Package',
-          long_description=readme,
-          long_description_content_type='text/x-rst',
-          python_requires='>=3.6',
-          install_requires=[
-              'wheel',
-              'numpy',
-              'scipy',
-              'scikit-learn!=0.22.0'
-          ],
-          extras_require={
-              'dask': [
-                  'dask[array]>=2.0.0',
-                  'dask[dataframe]>=2.0.0',
-                  'dask[distributed]>=2.0.0',
-                  'pandas',
-              ],
-          },
-          maintainer='Yu Shi',
-          maintainer_email='yushi2@microsoft.com',
-          zip_safe=False,
-          cmdclass={
-              'install': CustomInstall,
-              'install_lib': CustomInstallLib,
-              'bdist_wheel': CustomBdistWheel,
-              'sdist': CustomSdist,
-          },
-          packages=find_packages(),
-          include_package_data=True,
-          license='The MIT License (Microsoft)',
-          url='https://github.com/microsoft/LightGBM',
-          classifiers=['Development Status :: 5 - Production/Stable',
-                       'Intended Audience :: Science/Research',
-                       'License :: OSI Approved :: MIT License',
-                       'Natural Language :: English',
-                       'Operating System :: MacOS',
-                       'Operating System :: Microsoft :: Windows',
-                       'Operating System :: POSIX',
-                       'Operating System :: Unix',
-                       'Programming Language :: Python :: 3',
-                       'Programming Language :: Python :: 3.7',
-                       'Programming Language :: Python :: 3.8',
-                       'Programming Language :: Python :: 3.9',
-                       'Programming Language :: Python :: 3.10',
-                       'Topic :: Scientific/Engineering :: Artificial Intelligence',
-                       'Typing :: Typed'])