upgrade publish package and CI process to adapt to changes in LightGBM about finding libpath (#28)

Co-authored-by: Ubuntu <azureuser@chjinche-vm-1109.ze3cvvdrjxpufcudvnip4kozqd.cdmx.internal.cloudapp.net>

.ci/setup.sh

@@ -6,9 +6,10 @@ mkdir build
 # precompile mpi option.
 cd build && cmake ../ -DUSE_MPI=ON && make -j4 && cd ../ || exit -1
 # copy all shared libs to lightgbm python package directory.
-cp ./lib_custom_parser.so ${lgb_python_pkg_dir}/ && \
-cp ./src/lib_transform.so ${lgb_python_pkg_dir}/ && \
-cp ./external_libs/LightGBM/lib_lightgbm.so ${lgb_python_pkg_dir}/ || exit -1
+mkdir ${lgb_python_pkg_dir}/lightgbm/bin && \
+cp ./lib_custom_parser.so ${lgb_python_pkg_dir}/lightgbm/bin && \
+cp ./src/lib_transform.so ${lgb_python_pkg_dir}/lightgbm/bin && \
+cp ./external_libs/LightGBM/lib_lightgbm.so ${lgb_python_pkg_dir}/lightgbm/bin || exit -1
 # modify `basic.py` to load all libs first, or cannot find them when calling python interfaces.
 cp ${lgb_python_pkg_dir}/lightgbm/basic.py raw && cat ./scripts/load_precompiled_libs.py ${lgb_python_pkg_dir}/lightgbm/basic.py > tmp && cp tmp ${lgb_python_pkg_dir}/lightgbm/basic.py || exit -1
 # install python package

scripts/MANIFEST.in

@@ -0,0 +1 @@
+include lightgbm/bin/*.so

scripts/load_precompiled_libs.py

@@ -3,4 +3,5 @@ from pathlib import Path
 
 CUSTOM_PARSER_LIB_NAME = 'lib_custom_parser.so'
 for p in ['lib_transform.so', 'lib_lightgbm.so', CUSTOM_PARSER_LIB_NAME]:
-    ctypes.cdll.LoadLibrary(str(Path(__file__).resolve().parents[1] / p))
+    # due to changes in LightGBM libpath.py https://github.com/microsoft/LightGBM/pull/6192
+    ctypes.cdll.LoadLibrary(str(Path(__file__).resolve().parent / 'bin' / p))

scripts/publish_python_package.sh

@@ -5,17 +5,19 @@ mkdir build
 # precompile mpi option.
 cd build && cmake ../ -DUSE_MPI=ON && make -j4 && cd ../ || exit -1
 # copy all shared libs to lightgbm python package directory.
-cp ./lib_custom_parser.so ${lgb_python_pkg_dir}/ && \
-cp ./src/lib_transform.so ${lgb_python_pkg_dir}/ && \
-cp ./external_libs/LightGBM/lib_lightgbm.so ${lgb_python_pkg_dir}/ || exit -1
 # modify `basic.py` to load all libs first, or cannot find them when calling python interfaces.
-cp ${lgb_python_pkg_dir}/lightgbm/basic.py raw && cat ./scripts/load_precompiled_libs.py ${lgb_python_pkg_dir}/lightgbm/basic.py > tmp && cp tmp ${lgb_python_pkg_dir}/lightgbm/basic.py || exit -1
 mkdir compile
-cp -r ${lgb_python_pkg_dir}/*.so ./compile/ && cp -r ${lgb_python_pkg_dir}/lightgbm ./compile/ && cp ${lgb_python_pkg_dir}/MANIFEST.in ./compile/ || exit -1
-cp ./scripts/setup.py ./compile/ && cp ./scripts/README.rst ./compile && cp VERSION.txt ./compile || exit -1
+cp -r ${lgb_python_pkg_dir}/lightgbm ./compile/ || exit -1
+mkdir compile/lightgbm/bin
+cp ./lib_custom_parser.so ${lgb_python_pkg_dir}/lightgbm/bin && \
+cp ./src/lib_transform.so ${lgb_python_pkg_dir}/lightgbm/bin && \
+cp ./external_libs/LightGBM/lib_lightgbm.so ${lgb_python_pkg_dir}/lightgbm/bin || exit -1 
+cat ./scripts/load_precompiled_libs.py ./compile/lightgbm/basic.py > tmp && cp tmp ./compile/lightgbm/basic.py || exit -1
+rm -rf tmp
+cp ./scripts/setup.py ./compile/ && cp ./scripts/README.rst ./compile && cp VERSION.txt ./compile && cp ./scripts/MANIFEST.in ./compile || exit -1
 # pack wheel package.
 cd compile && python setup.py bdist_wheel --precompile && cd ../ || exit -1
 # upload package to your pypi, use testpypi as an example.
 twine upload --repository pypi compile/dist/* || exit -1
 # revert changes.
-mv raw ${lgb_python_pkg_dir}/lightgbm/basic.py && rm -rf raw tmp ${lgb_python_pkg_dir}/lightgbm/*.so build compile || exit -1
+rm -rf ${lgb_python_pkg_dir}/lightgbm/*.so build compile || exit -1

scripts/setup.py

@@ -11,9 +11,28 @@ from typing import List, Optional, Union
 
 from setuptools import find_packages, setup
 from setuptools.command.install import install
+from wheel.bdist_wheel import bdist_wheel
 
 sys.path.insert(0, '../external_libs/LightGBM/python-package')
-from setup import LIGHTGBM_OPTIONS, CustomBdistWheel, clear_path
+
+LIGHTGBM_OPTIONS = [
+    ('mingw', 'm', 'Compile with MinGW'),
+    ('integrated-opencl', None, 'Compile integrated OpenCL version'),
+    ('gpu', 'g', 'Compile GPU version'),
+    ('cuda', None, 'Compile CUDA version'),
+    ('mpi', None, 'Compile MPI version'),
+    ('nomp', None, 'Compile version without OpenMP support'),
+    ('hdfs', 'h', 'Compile HDFS version'),
+    ('bit32', None, 'Compile 32-bit version'),
+    ('precompile', 'p', 'Use precompiled library'),
+    ('time-costs', None, 'Output time costs for different internal routines'),
+    ('boost-root=', None, 'Boost preferred installation prefix'),
+    ('boost-dir=', None, 'Directory with Boost package configuration file'),
+    ('boost-include-dir=', None, 'Directory containing Boost headers'),
+    ('boost-librarydir=', None, 'Preferred Boost library directory'),
+    ('opencl-include-dir=', None, 'OpenCL include directory'),
+    ('opencl-library=', None, 'Path to OpenCL library')
+]
 
 
 class CustomInstall(install):
@@ -55,6 +74,52 @@ class CustomInstall(install):
             LOG_PATH.unlink()
 
 
+class CustomBdistWheel(bdist_wheel):
+
+    user_options = bdist_wheel.user_options + LIGHTGBM_OPTIONS
+
+    def initialize_options(self) -> None:
+        bdist_wheel.initialize_options(self)
+        self.mingw = False
+        self.integrated_opencl = False
+        self.gpu = False
+        self.cuda = False
+        self.boost_root = None
+        self.boost_dir = None
+        self.boost_include_dir = None
+        self.boost_librarydir = None
+        self.opencl_include_dir = None
+        self.opencl_library = None
+        self.mpi = False
+        self.hdfs = False
+        self.precompile = False
+        self.time_costs = False
+        self.nomp = False
+        self.bit32 = False
+
+    def finalize_options(self) -> None:
+        bdist_wheel.finalize_options(self)
+
+        install = self.reinitialize_command('install')
+
+        install.mingw = self.mingw
+        install.integrated_opencl = self.integrated_opencl
+        install.gpu = self.gpu
+        install.cuda = self.cuda
+        install.boost_root = self.boost_root
+        install.boost_dir = self.boost_dir
+        install.boost_include_dir = self.boost_include_dir
+        install.boost_librarydir = self.boost_librarydir
+        install.opencl_include_dir = self.opencl_include_dir
+        install.opencl_library = self.opencl_library
+        install.mpi = self.mpi
+        install.hdfs = self.hdfs
+        install.precompile = self.precompile
+        install.time_costs = self.time_costs
+        install.nomp = self.nomp
+        install.bit32 = self.bit32
+
+
 if __name__ == '__main__':
     CURRENT_DIR = Path(__file__).absolute().parent
     LOG_PATH = Path.home() / 'LightGBM_compilation.log'