зеркало из https://github.com/microsoft/LightGBM.git
[python] added option to install python-package with mpi version and possibility to pass CMake options (#1034)
* fixed hierarchy of installation guide sections; made installation with proto support more verbose
* added option to install with protobuf support to python package
* fixed pylint
* added option to install MPI version to python package
* added tests for python-package installation with options --proto and --mpi
* hotfix
* removed unnecessary note from docs
* removed unnecessary line from travis script
* reverted proto
* Revert "reverted proto"
This reverts commit 5203448bb9.
* reverted protobuf from python-package
* added options for cmake in python-package
* fixed typos
* fixed options readability
* reworked installation logging
* fixed docs according to review comments
* fixed pylint
* fixed space in log path
This commit is contained in:
Nikita Titov
Guolin Ke
Родитель
2e80196727
Коммит
c97abff407
|
|
@ -19,6 +19,8 @@ if [[ $TRAVIS_OS_NAME == "osx" ]]; then
|
|||
export CC=gcc-7
|
||||
fi
|
||||
|
||||
LGB_VER=$(head -n 1 VERSION.txt)
|
||||
|
||||
conda create -q -n test-env python=$PYTHON_VERSION
|
||||
source activate test-env
|
||||
|
||||
|
|
@ -65,13 +67,11 @@ fi
|
|||
conda install numpy nose scipy scikit-learn pandas matplotlib pytest
|
||||
|
||||
if [[ ${TASK} == "sdist" ]]; then
|
||||
LGB_VER=$(head -n 1 VERSION.txt)
|
||||
cd $TRAVIS_BUILD_DIR/python-package && python setup.py sdist || exit -1
|
||||
cd $TRAVIS_BUILD_DIR/python-package/dist && pip install lightgbm-$LGB_VER.tar.gz -v || exit -1
|
||||
cd $TRAVIS_BUILD_DIR && pytest tests/python_package_test || exit -1
|
||||
exit 0
|
||||
elif [[ ${TASK} == "bdist" ]]; then
|
||||
LGB_VER=$(head -n 1 VERSION.txt)
|
||||
if [[ $TRAVIS_OS_NAME == "osx" ]]; then
|
||||
cd $TRAVIS_BUILD_DIR/python-package && python setup.py bdist_wheel --plat-name=macosx --universal || exit -1
|
||||
mv dist/lightgbm-${LGB_VER}-py2.py3-none-macosx.whl dist/lightgbm-${LGB_VER}-py2.py3-none-macosx_10_9_x86_64.macosx_10_10_x86_64.macosx_10_11_x86_64.macosx_10_12_x86_64.whl
|
||||
|
|
@ -88,7 +88,6 @@ if [[ ${TASK} == "gpu" ]]; then
|
|||
if [[ ${METHOD} == "pip" ]]; then
|
||||
export PATH="$AMDAPPSDK/include/:$PATH"
|
||||
export BOOST_ROOT="$HOME/miniconda/envs/test-env/"
|
||||
LGB_VER=$(head -n 1 VERSION.txt)
|
||||
sed -i 's/const std::string kDefaultDevice = "cpu";/const std::string kDefaultDevice = "gpu";/' ../include/LightGBM/config.h
|
||||
cd $TRAVIS_BUILD_DIR/python-package && python setup.py sdist || exit -1
|
||||
cd $TRAVIS_BUILD_DIR/python-package/dist && pip install lightgbm-$LGB_VER.tar.gz -v --install-option=--gpu || exit -1
|
||||
|
|
@ -100,6 +99,9 @@ fi
|
|||
mkdir build && cd build
|
||||
|
||||
if [[ ${TASK} == "mpi" ]]; then
|
||||
cd $TRAVIS_BUILD_DIR/python-package && python setup.py sdist || exit -1
|
||||
cd $TRAVIS_BUILD_DIR/python-package/dist && pip install lightgbm-$LGB_VER.tar.gz -v --install-option=--mpi || exit -1
|
||||
cd $TRAVIS_BUILD_DIR/build
|
||||
cmake -DUSE_MPI=ON ..
|
||||
elif [[ ${TASK} == "gpu" ]]; then
|
||||
cmake -DUSE_GPU=ON -DBOOST_ROOT="$HOME/miniconda/envs/test-env/" -DOpenCL_INCLUDE_DIR=$AMDAPPSDK/include/ ..
|
||||
|
|
|
|||
|
|
@ -61,8 +61,8 @@ Now we are ready to checkout LightGBM and compile it with GPU support:
|
|||
cd LightGBM
|
||||
mkdir build ; cd build
|
||||
cmake -DUSE_GPU=1 ..
|
||||
# if you have installed the NVIDIA OpenGL, please using following instead
|
||||
# sudo cmake -DUSE_GPU=1 -DOpenCL_LIBRARY=/usr/local/cuda/lib64/libOpenCL.so -OpenCL_INCLUDE_DIR=/usr/local/cuda/include/ ..
|
||||
# if you have installed NVIDIA CUDA to a customized location, you should specify paths to OpenCL headers and library like the following:
|
||||
# cmake -DUSE_GPU=1 -DOpenCL_LIBRARY=/usr/local/cuda/lib64/libOpenCL.so -DOpenCL_INCLUDE_DIR=/usr/local/cuda/include/ ..
|
||||
make -j$(nproc)
|
||||
cd ..
|
||||
|
||||
|
|
|
|||
|
|
@ -64,7 +64,7 @@ MinGW64
|
|||
|
||||
The exe and dll files will be in ``LightGBM/`` folder.
|
||||
|
||||
**Note**: you may need to run the ``cmake -G "MinGW Makefiles" ..`` one more time if met ``sh.exe was found in your PATH`` error.
|
||||
**Note**: You may need to run the ``cmake -G "MinGW Makefiles" ..`` one more time if met ``sh.exe was found in your PATH`` error.
|
||||
|
||||
Also you may want to reed `gcc Tips <./gcc-Tips.rst>`__.
|
||||
|
||||
|
|
@ -161,7 +161,7 @@ From Command Line
|
|||
|
||||
The exe and dll files will be in ``LightGBM/Release`` folder.
|
||||
|
||||
**Note**: Build MPI version by **MinGW** is not supported due to the miss of MPI library in it.
|
||||
**Note**: Building MPI version by **MinGW** is not supported due to the miss of MPI library in it.
|
||||
|
||||
Linux
|
||||
^^^^^
|
||||
|
|
@ -226,8 +226,8 @@ To build LightGBM GPU version, run the following commands:
|
|||
git clone --recursive https://github.com/Microsoft/LightGBM ; cd LightGBM
|
||||
mkdir build ; cd build
|
||||
cmake -DUSE_GPU=1 ..
|
||||
# if you have installed the NVIDIA OpenGL, please use following command instead
|
||||
# sudo cmake -DUSE_GPU=1 -DOpenCL_LIBRARY=/usr/local/cuda/lib64/libOpenCL.so -OpenCL_INCLUDE_DIR=/usr/local/cuda/include/ ..
|
||||
# if you have installed NVIDIA CUDA to a customized location, you should specify paths to OpenCL headers and library like the following:
|
||||
# cmake -DUSE_GPU=1 -DOpenCL_LIBRARY=/usr/local/cuda/lib64/libOpenCL.so -DOpenCL_INCLUDE_DIR=/usr/local/cuda/include/ ..
|
||||
make -j4
|
||||
|
||||
Windows
|
||||
|
|
@ -249,7 +249,7 @@ Following procedure is for the MSVC (Microsoft Visual C++) build.
|
|||
|
||||
3. Install `Boost Binary`_.
|
||||
|
||||
**Note**: match your Visual C++ version:
|
||||
**Note**: Match your Visual C++ version:
|
||||
|
||||
Visual Studio 2015 -> ``msvc-14.0-64.exe``,
|
||||
|
||||
|
|
|
|||
|
|
@ -38,6 +38,17 @@ For macOS users, you need to specify compilers by runnig ``export CXX=g++-7 CC=g
|
|||
|
||||
For Windows users, Visual Studio (or `MS Build <https://www.visualstudio.com/downloads/>`_) is needed. If you get any errors during installation, you may need to install `CMake <https://cmake.org/>`_ (version 3.8 or higher).
|
||||
|
||||
Build MPI Version
|
||||
~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. code:: sh
|
||||
|
||||
pip install lightgbm --install-option=--mpi
|
||||
|
||||
For Windows users, compilation with MinGW-w64 is not supported and `CMake <https://cmake.org/>`_ (version 3.8 or higher) is strongly required in this case.
|
||||
|
||||
Note: MPI libraries are needed: details for installation can be found in `Installation Guide <https://github.com/Microsoft/LightGBM/blob/master/docs/Installation-Guide.rst#build-mpi-version>`__.
|
||||
|
||||
Build GPU Version
|
||||
~~~~~~~~~~~~~~~~~
|
||||
|
||||
|
|
@ -49,6 +60,28 @@ For Windows users, `CMake <https://cmake.org/>`_ (version 3.8 or higher) is stro
|
|||
|
||||
Note: Boost and OpenCL are needed: details for installation can be found in `Installation Guide <https://github.com/Microsoft/LightGBM/blob/master/docs/Installation-Guide.rst#build-gpu-version>`__. You need to add ``OpenCL_INCLUDE_DIR`` to the environmental variable **'PATH'** and export ``BOOST_ROOT`` before installation.
|
||||
|
||||
Also you may pass options to CMake via pip options, like
|
||||
|
||||
.. code:: sh
|
||||
|
||||
pip install lightgbm --install-option=--gpu --install-option="--opencl-include-dir=/usr/local/cuda/include/" --install-option="--opencl-library=/usr/local/cuda/lib64/libOpenCL.so"
|
||||
|
||||
All available options:
|
||||
|
||||
- boost-root
|
||||
|
||||
- boost-dir
|
||||
|
||||
- boost-include-dir
|
||||
|
||||
- boost-librarydir
|
||||
|
||||
- opencl-include-dir
|
||||
|
||||
- opencl-library
|
||||
|
||||
For more details see `FindBoost <https://cmake.org/cmake/help/v3.8/module/FindBoost.html>`__ and `FindOpenCL <https://cmake.org/cmake/help/v3.8/module/FindOpenCL.html>`__.
|
||||
|
||||
Build with MinGW-w64 on Windows
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
|
|
@ -74,9 +107,11 @@ For Windows users, Visual Studio (or `MS Build <https://www.visualstudio.com/dow
|
|||
|
||||
Note: ``sudo`` (or administrator rights in Windows) may be needed to perform the command.
|
||||
|
||||
Run ``python setup.py install --mpi`` to enable MPI support. For Windows users, compilation with MinGW-w64 is not supported and `CMake <https://cmake.org/>`_ (version 3.8 or higher) is strongly required in this case. MPI libraries are needed: details for installation can be found in `Installation Guide <https://github.com/Microsoft/LightGBM/blob/master/docs/Installation-Guide.rst#build-mpi-version>`__.
|
||||
|
||||
Run ``python setup.py install --mingw`` if you want to use MinGW-w64 on Windows instead of Visual Studio. `CMake <https://cmake.org/>`_ and `MinGW-w64 <https://mingw-w64.org/>`_ should be installed first.
|
||||
|
||||
Run ``python setup.py install --gpu`` to enable GPU support. For Windows users, `CMake <https://cmake.org/>`_ (version 3.8 or higher) is strongly required in this case. Boost and OpenCL are needed: details for installation can be found in `Installation Guide <https://github.com/Microsoft/LightGBM/blob/master/docs/Installation-Guide.rst#build-gpu-version>`__.
|
||||
Run ``python setup.py install --gpu`` to enable GPU support. For Windows users, `CMake <https://cmake.org/>`_ (version 3.8 or higher) is strongly required in this case. Boost and OpenCL are needed: details for installation can be found in `Installation Guide <https://github.com/Microsoft/LightGBM/blob/master/docs/Installation-Guide.rst#build-gpu-version>`__. You can pass additional options to CMake: ``python setup.py install --gpu --opencl-include-dir=/usr/local/cuda/include/``, see `Build GPU Version <#build-gpu-version>`__ for complete list of them.
|
||||
|
||||
If you get any errors during installation or due to any other reason, you may want to build dynamic library from sources by any method you prefer (see `Installation Guide <https://github.com/Microsoft/LightGBM/blob/master/docs/Installation-Guide.rst>`__) and then run ``python setup.py install --precompile``.
|
||||
|
||||
|
|
|
|||
|
|
@ -1420,7 +1420,7 @@ class Booster(object):
|
|||
self.network = True
|
||||
|
||||
def free_network(self):
|
||||
"""Free Network."""
|
||||
"""Free network."""
|
||||
_safe_call(_LIB.LGBM_NetworkFree())
|
||||
self.network = False
|
||||
|
||||
|
|
|
|||
|
|
@ -65,16 +65,23 @@ def clear_path(path):
|
|||
|
||||
def silent_call(cmd, raise_error=False, error_msg=''):
|
||||
try:
|
||||
with open(os.devnull, "w") as shut_up:
|
||||
subprocess.check_output(cmd, stderr=shut_up)
|
||||
return 0
|
||||
except Exception:
|
||||
output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
|
||||
with open(path_log, "ab") as log:
|
||||
log.write(output)
|
||||
return 0
|
||||
except Exception as err:
|
||||
if isinstance(err, subprocess.CalledProcessError):
|
||||
with open(path_log, "ab") as log:
|
||||
log.write(err.output)
|
||||
if raise_error:
|
||||
raise Exception(error_msg)
|
||||
raise Exception("\n".join((error_msg, log_notice)))
|
||||
return 1
|
||||
|
||||
|
||||
def compile_cpp(use_mingw=False, use_gpu=False):
|
||||
def compile_cpp(use_mingw=False, use_gpu=False, use_mpi=False,
|
||||
boost_root=None, boost_dir=None, boost_include_dir=None,
|
||||
boost_librarydir=None, opencl_include_dir=None,
|
||||
opencl_library=None):
|
||||
|
||||
if os.path.exists("build_cpp"):
|
||||
shutil.rmtree("build_cpp")
|
||||
|
|
@ -86,11 +93,27 @@ def compile_cpp(use_mingw=False, use_gpu=False):
|
|||
cmake_cmd = ["cmake", "../compile/"]
|
||||
if use_gpu:
|
||||
cmake_cmd.append("-DUSE_GPU=ON")
|
||||
if boost_root:
|
||||
cmake_cmd.append("-DBOOST_ROOT={0}".format(boost_root))
|
||||
if boost_dir:
|
||||
cmake_cmd.append("-DBoost_DIR={0}".format(boost_dir))
|
||||
if boost_include_dir:
|
||||
cmake_cmd.append("-DBoost_INCLUDE_DIR={0}".format(boost_include_dir))
|
||||
if boost_librarydir:
|
||||
cmake_cmd.append("-DBOOST_LIBRARYDIR={0}".format(boost_librarydir))
|
||||
if opencl_include_dir:
|
||||
cmake_cmd.append("-DOpenCL_INCLUDE_DIR={0}".format(opencl_include_dir))
|
||||
if opencl_library:
|
||||
cmake_cmd.append("-DOpenCL_LIBRARY={0}".format(opencl_library))
|
||||
if use_mpi:
|
||||
cmake_cmd.append("-DUSE_MPI=ON")
|
||||
if os.name == "nt":
|
||||
if use_mingw:
|
||||
if use_mpi:
|
||||
raise Exception('MPI version cannot be compiled by MinGW due to the miss of MPI library in it')
|
||||
logger.info("Starting to compile with CMake and MinGW.")
|
||||
silent_call(cmake_cmd + ["-G", "MinGW Makefiles"], raise_error=True,
|
||||
error_msg='Please install CMake first')
|
||||
error_msg='Please install CMake and all required dependencies first')
|
||||
silent_call(["mingw32-make.exe", "_lightgbm"], raise_error=True,
|
||||
error_msg='Please install MinGW first')
|
||||
else:
|
||||
|
|
@ -120,12 +143,13 @@ def compile_cpp(use_mingw=False, use_gpu=False):
|
|||
else:
|
||||
clear_path("./")
|
||||
if status != 0:
|
||||
raise Exception('Please install Visual Studio or MS Build first')
|
||||
raise Exception("\n".join(('Please install Visual Studio or MS Build and all required dependencies first',
|
||||
log_notice)))
|
||||
silent_call(["cmake", "--build", ".", "--target", "_lightgbm", "--config", "Release"], raise_error=True,
|
||||
error_msg='Please install CMake first')
|
||||
else: # Linux, Darwin (OS X), etc.
|
||||
logger.info("Starting to compile with CMake.")
|
||||
silent_call(cmake_cmd, raise_error=True, error_msg='Please install CMake first')
|
||||
silent_call(cmake_cmd, raise_error=True, error_msg='Please install CMake and all required dependencies first')
|
||||
silent_call(["make", "_lightgbm"], raise_error=True,
|
||||
error_msg='An error has occurred while building lightgbm library file')
|
||||
os.chdir("..")
|
||||
|
|
@ -145,22 +169,42 @@ class CustomInstallLib(install_lib):
|
|||
class CustomInstall(install):
|
||||
|
||||
user_options = install.user_options + [
|
||||
('mingw', 'm', 'compile with mingw'),
|
||||
('gpu', 'g', 'compile gpu version'),
|
||||
('precompile', 'p', 'use precompiled library')
|
||||
('mingw', 'm', 'Compile with MinGW'),
|
||||
('gpu', 'g', 'Compile GPU version'),
|
||||
('mpi', None, 'Compile MPI version'),
|
||||
('precompile', 'p', 'Use precompiled library'),
|
||||
('boost-root=', None, 'Boost preferred installation prefix'),
|
||||
('boost-dir=', None, 'Directory with Boost package configuration file'),
|
||||
('boost-include-dir=', None, 'Directory containing Boost headers'),
|
||||
('boost-librarydir=', None, 'Preferred Boost library directory'),
|
||||
('opencl-include-dir=', None, 'OpenCL include directory'),
|
||||
('opencl-library=', None, 'Path to OpenCL library')
|
||||
]
|
||||
|
||||
def initialize_options(self):
|
||||
install.initialize_options(self)
|
||||
self.mingw = 0
|
||||
self.gpu = 0
|
||||
self.boost_root = None
|
||||
self.boost_dir = None
|
||||
self.boost_include_dir = None
|
||||
self.boost_librarydir = None
|
||||
self.opencl_include_dir = None
|
||||
self.opencl_library = None
|
||||
self.mpi = 0
|
||||
self.precompile = 0
|
||||
|
||||
def run(self):
|
||||
open(path_log, 'wb').close()
|
||||
if not self.precompile:
|
||||
copy_files(use_gpu=self.gpu)
|
||||
compile_cpp(use_mingw=self.mingw, use_gpu=self.gpu)
|
||||
compile_cpp(use_mingw=self.mingw, use_gpu=self.gpu, use_mpi=self.mpi,
|
||||
boost_root=self.boost_root, boost_dir=self.boost_dir,
|
||||
boost_include_dir=self.boost_include_dir, boost_librarydir=self.boost_librarydir,
|
||||
opencl_include_dir=self.opencl_include_dir, opencl_library=self.opencl_library)
|
||||
install.run(self)
|
||||
if os.path.isfile(path_log):
|
||||
os.remove(path_log)
|
||||
|
||||
|
||||
class CustomSdist(sdist):
|
||||
|
|
@ -181,9 +225,11 @@ class CustomSdist(sdist):
|
|||
|
||||
if __name__ == "__main__":
|
||||
if (8 * struct.calcsize("P")) != 64:
|
||||
raise Exception('Cannot install LightGBM in 32-bit python, please use 64-bit python instead.')
|
||||
raise Exception('Cannot install LightGBM in 32-bit Python, please use 64-bit python instead.')
|
||||
|
||||
dir_path = os.path.dirname(os.path.realpath(__file__))
|
||||
path_log = os.path.join(os.path.expanduser('~'), 'LightGBM_compilation.log')
|
||||
log_notice = "The full version of error log was saved into {0}".format(path_log)
|
||||
if os.path.isfile(os.path.join('..', 'VERSION.txt')):
|
||||
distutils.file_util.copy_file(os.path.join('..', 'VERSION.txt'),
|
||||
os.path.join('.', 'lightgbm'))
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче