diff --git a/.travis/test.sh b/.travis/test.sh index e5193a3e3..17a52d507 100644 --- a/.travis/test.sh +++ b/.travis/test.sh @@ -19,6 +19,8 @@ if [[ $TRAVIS_OS_NAME == "osx" ]]; then export CC=gcc-7 fi +LGB_VER=$(head -n 1 VERSION.txt) + conda create -q -n test-env python=$PYTHON_VERSION source activate test-env @@ -65,13 +67,11 @@ fi conda install numpy nose scipy scikit-learn pandas matplotlib pytest if [[ ${TASK} == "sdist" ]]; then - LGB_VER=$(head -n 1 VERSION.txt) cd $TRAVIS_BUILD_DIR/python-package && python setup.py sdist || exit -1 cd $TRAVIS_BUILD_DIR/python-package/dist && pip install lightgbm-$LGB_VER.tar.gz -v || exit -1 cd $TRAVIS_BUILD_DIR && pytest tests/python_package_test || exit -1 exit 0 elif [[ ${TASK} == "bdist" ]]; then - LGB_VER=$(head -n 1 VERSION.txt) if [[ $TRAVIS_OS_NAME == "osx" ]]; then cd $TRAVIS_BUILD_DIR/python-package && python setup.py bdist_wheel --plat-name=macosx --universal || exit -1 mv dist/lightgbm-${LGB_VER}-py2.py3-none-macosx.whl dist/lightgbm-${LGB_VER}-py2.py3-none-macosx_10_9_x86_64.macosx_10_10_x86_64.macosx_10_11_x86_64.macosx_10_12_x86_64.whl @@ -88,7 +88,6 @@ if [[ ${TASK} == "gpu" ]]; then if [[ ${METHOD} == "pip" ]]; then export PATH="$AMDAPPSDK/include/:$PATH" export BOOST_ROOT="$HOME/miniconda/envs/test-env/" - LGB_VER=$(head -n 1 VERSION.txt) sed -i 's/const std::string kDefaultDevice = "cpu";/const std::string kDefaultDevice = "gpu";/' ../include/LightGBM/config.h cd $TRAVIS_BUILD_DIR/python-package && python setup.py sdist || exit -1 cd $TRAVIS_BUILD_DIR/python-package/dist && pip install lightgbm-$LGB_VER.tar.gz -v --install-option=--gpu || exit -1 @@ -100,6 +99,9 @@ fi mkdir build && cd build if [[ ${TASK} == "mpi" ]]; then + cd $TRAVIS_BUILD_DIR/python-package && python setup.py sdist || exit -1 + cd $TRAVIS_BUILD_DIR/python-package/dist && pip install lightgbm-$LGB_VER.tar.gz -v --install-option=--mpi || exit -1 + cd $TRAVIS_BUILD_DIR/build cmake -DUSE_MPI=ON .. elif [[ ${TASK} == "gpu" ]]; then cmake -DUSE_GPU=ON -DBOOST_ROOT="$HOME/miniconda/envs/test-env/" -DOpenCL_INCLUDE_DIR=$AMDAPPSDK/include/ .. diff --git a/docs/GPU-Tutorial.rst b/docs/GPU-Tutorial.rst index e32b77517..1057392ce 100644 --- a/docs/GPU-Tutorial.rst +++ b/docs/GPU-Tutorial.rst @@ -61,8 +61,8 @@ Now we are ready to checkout LightGBM and compile it with GPU support: cd LightGBM mkdir build ; cd build cmake -DUSE_GPU=1 .. -   # if you have installed the NVIDIA OpenGL, please using following instead - # sudo cmake -DUSE_GPU=1 -DOpenCL_LIBRARY=/usr/local/cuda/lib64/libOpenCL.so -OpenCL_INCLUDE_DIR=/usr/local/cuda/include/ .. +   # if you have installed NVIDIA CUDA to a customized location, you should specify paths to OpenCL headers and library like the following: + # cmake -DUSE_GPU=1 -DOpenCL_LIBRARY=/usr/local/cuda/lib64/libOpenCL.so -DOpenCL_INCLUDE_DIR=/usr/local/cuda/include/ .. make -j$(nproc) cd .. diff --git a/docs/Installation-Guide.rst b/docs/Installation-Guide.rst index 0f0ccd122..5be9fcf2d 100644 --- a/docs/Installation-Guide.rst +++ b/docs/Installation-Guide.rst @@ -64,7 +64,7 @@ MinGW64 The exe and dll files will be in ``LightGBM/`` folder. -**Note**: you may need to run the ``cmake -G "MinGW Makefiles" ..`` one more time if met ``sh.exe was found in your PATH`` error. +**Note**: You may need to run the ``cmake -G "MinGW Makefiles" ..`` one more time if met ``sh.exe was found in your PATH`` error. Also you may want to reed `gcc Tips <./gcc-Tips.rst>`__. @@ -161,7 +161,7 @@ From Command Line The exe and dll files will be in ``LightGBM/Release`` folder. -**Note**: Build MPI version by **MinGW** is not supported due to the miss of MPI library in it. +**Note**: Building MPI version by **MinGW** is not supported due to the miss of MPI library in it. Linux ^^^^^ @@ -226,8 +226,8 @@ To build LightGBM GPU version, run the following commands: git clone --recursive https://github.com/Microsoft/LightGBM ; cd LightGBM mkdir build ; cd build cmake -DUSE_GPU=1 .. - # if you have installed the NVIDIA OpenGL, please use following command instead - # sudo cmake -DUSE_GPU=1 -DOpenCL_LIBRARY=/usr/local/cuda/lib64/libOpenCL.so -OpenCL_INCLUDE_DIR=/usr/local/cuda/include/ .. + # if you have installed NVIDIA CUDA to a customized location, you should specify paths to OpenCL headers and library like the following: + # cmake -DUSE_GPU=1 -DOpenCL_LIBRARY=/usr/local/cuda/lib64/libOpenCL.so -DOpenCL_INCLUDE_DIR=/usr/local/cuda/include/ .. make -j4 Windows @@ -249,7 +249,7 @@ Following procedure is for the MSVC (Microsoft Visual C++) build. 3. Install `Boost Binary`_. - **Note**: match your Visual C++ version: + **Note**: Match your Visual C++ version: Visual Studio 2015 -> ``msvc-14.0-64.exe``, diff --git a/python-package/README.rst b/python-package/README.rst index b667681f2..c98809ee9 100644 --- a/python-package/README.rst +++ b/python-package/README.rst @@ -38,6 +38,17 @@ For macOS users, you need to specify compilers by runnig ``export CXX=g++-7 CC=g For Windows users, Visual Studio (or `MS Build `_) is needed. If you get any errors during installation, you may need to install `CMake `_ (version 3.8 or higher). +Build MPI Version +~~~~~~~~~~~~~~~~~ + +.. code:: sh + + pip install lightgbm --install-option=--mpi + +For Windows users, compilation with MinGW-w64 is not supported and `CMake `_ (version 3.8 or higher) is strongly required in this case. + +Note: MPI libraries are needed: details for installation can be found in `Installation Guide `__. + Build GPU Version ~~~~~~~~~~~~~~~~~ @@ -49,6 +60,28 @@ For Windows users, `CMake `_ (version 3.8 or higher) is stro Note: Boost and OpenCL are needed: details for installation can be found in `Installation Guide `__. You need to add ``OpenCL_INCLUDE_DIR`` to the environmental variable **'PATH'** and export ``BOOST_ROOT`` before installation. +Also you may pass options to CMake via pip options, like + +.. code:: sh + + pip install lightgbm --install-option=--gpu --install-option="--opencl-include-dir=/usr/local/cuda/include/" --install-option="--opencl-library=/usr/local/cuda/lib64/libOpenCL.so" + +All available options: + +- boost-root + +- boost-dir + +- boost-include-dir + +- boost-librarydir + +- opencl-include-dir + +- opencl-library + +For more details see `FindBoost `__ and `FindOpenCL `__. + Build with MinGW-w64 on Windows ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -74,9 +107,11 @@ For Windows users, Visual Studio (or `MS Build `_ (version 3.8 or higher) is strongly required in this case. MPI libraries are needed: details for installation can be found in `Installation Guide `__. + Run ``python setup.py install --mingw`` if you want to use MinGW-w64 on Windows instead of Visual Studio. `CMake `_ and `MinGW-w64 `_ should be installed first. -Run ``python setup.py install --gpu`` to enable GPU support. For Windows users, `CMake `_ (version 3.8 or higher) is strongly required in this case. Boost and OpenCL are needed: details for installation can be found in `Installation Guide `__. +Run ``python setup.py install --gpu`` to enable GPU support. For Windows users, `CMake `_ (version 3.8 or higher) is strongly required in this case. Boost and OpenCL are needed: details for installation can be found in `Installation Guide `__. You can pass additional options to CMake: ``python setup.py install --gpu --opencl-include-dir=/usr/local/cuda/include/``, see `Build GPU Version <#build-gpu-version>`__ for complete list of them. If you get any errors during installation or due to any other reason, you may want to build dynamic library from sources by any method you prefer (see `Installation Guide `__) and then run ``python setup.py install --precompile``. diff --git a/python-package/lightgbm/basic.py b/python-package/lightgbm/basic.py index e741b612b..1d896f4b9 100644 --- a/python-package/lightgbm/basic.py +++ b/python-package/lightgbm/basic.py @@ -1420,7 +1420,7 @@ class Booster(object): self.network = True def free_network(self): - """Free Network.""" + """Free network.""" _safe_call(_LIB.LGBM_NetworkFree()) self.network = False diff --git a/python-package/setup.py b/python-package/setup.py index 7d2e2830a..46cda9218 100644 --- a/python-package/setup.py +++ b/python-package/setup.py @@ -65,16 +65,23 @@ def clear_path(path): def silent_call(cmd, raise_error=False, error_msg=''): try: - with open(os.devnull, "w") as shut_up: - subprocess.check_output(cmd, stderr=shut_up) - return 0 - except Exception: + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) + with open(path_log, "ab") as log: + log.write(output) + return 0 + except Exception as err: + if isinstance(err, subprocess.CalledProcessError): + with open(path_log, "ab") as log: + log.write(err.output) if raise_error: - raise Exception(error_msg) + raise Exception("\n".join((error_msg, log_notice))) return 1 -def compile_cpp(use_mingw=False, use_gpu=False): +def compile_cpp(use_mingw=False, use_gpu=False, use_mpi=False, + boost_root=None, boost_dir=None, boost_include_dir=None, + boost_librarydir=None, opencl_include_dir=None, + opencl_library=None): if os.path.exists("build_cpp"): shutil.rmtree("build_cpp") @@ -86,11 +93,27 @@ def compile_cpp(use_mingw=False, use_gpu=False): cmake_cmd = ["cmake", "../compile/"] if use_gpu: cmake_cmd.append("-DUSE_GPU=ON") + if boost_root: + cmake_cmd.append("-DBOOST_ROOT={0}".format(boost_root)) + if boost_dir: + cmake_cmd.append("-DBoost_DIR={0}".format(boost_dir)) + if boost_include_dir: + cmake_cmd.append("-DBoost_INCLUDE_DIR={0}".format(boost_include_dir)) + if boost_librarydir: + cmake_cmd.append("-DBOOST_LIBRARYDIR={0}".format(boost_librarydir)) + if opencl_include_dir: + cmake_cmd.append("-DOpenCL_INCLUDE_DIR={0}".format(opencl_include_dir)) + if opencl_library: + cmake_cmd.append("-DOpenCL_LIBRARY={0}".format(opencl_library)) + if use_mpi: + cmake_cmd.append("-DUSE_MPI=ON") if os.name == "nt": if use_mingw: + if use_mpi: + raise Exception('MPI version cannot be compiled by MinGW due to the miss of MPI library in it') logger.info("Starting to compile with CMake and MinGW.") silent_call(cmake_cmd + ["-G", "MinGW Makefiles"], raise_error=True, - error_msg='Please install CMake first') + error_msg='Please install CMake and all required dependencies first') silent_call(["mingw32-make.exe", "_lightgbm"], raise_error=True, error_msg='Please install MinGW first') else: @@ -120,12 +143,13 @@ def compile_cpp(use_mingw=False, use_gpu=False): else: clear_path("./") if status != 0: - raise Exception('Please install Visual Studio or MS Build first') + raise Exception("\n".join(('Please install Visual Studio or MS Build and all required dependencies first', + log_notice))) silent_call(["cmake", "--build", ".", "--target", "_lightgbm", "--config", "Release"], raise_error=True, error_msg='Please install CMake first') else: # Linux, Darwin (OS X), etc. logger.info("Starting to compile with CMake.") - silent_call(cmake_cmd, raise_error=True, error_msg='Please install CMake first') + silent_call(cmake_cmd, raise_error=True, error_msg='Please install CMake and all required dependencies first') silent_call(["make", "_lightgbm"], raise_error=True, error_msg='An error has occurred while building lightgbm library file') os.chdir("..") @@ -145,22 +169,42 @@ class CustomInstallLib(install_lib): class CustomInstall(install): user_options = install.user_options + [ - ('mingw', 'm', 'compile with mingw'), - ('gpu', 'g', 'compile gpu version'), - ('precompile', 'p', 'use precompiled library') + ('mingw', 'm', 'Compile with MinGW'), + ('gpu', 'g', 'Compile GPU version'), + ('mpi', None, 'Compile MPI version'), + ('precompile', 'p', 'Use precompiled library'), + ('boost-root=', None, 'Boost preferred installation prefix'), + ('boost-dir=', None, 'Directory with Boost package configuration file'), + ('boost-include-dir=', None, 'Directory containing Boost headers'), + ('boost-librarydir=', None, 'Preferred Boost library directory'), + ('opencl-include-dir=', None, 'OpenCL include directory'), + ('opencl-library=', None, 'Path to OpenCL library') ] def initialize_options(self): install.initialize_options(self) self.mingw = 0 self.gpu = 0 + self.boost_root = None + self.boost_dir = None + self.boost_include_dir = None + self.boost_librarydir = None + self.opencl_include_dir = None + self.opencl_library = None + self.mpi = 0 self.precompile = 0 def run(self): + open(path_log, 'wb').close() if not self.precompile: copy_files(use_gpu=self.gpu) - compile_cpp(use_mingw=self.mingw, use_gpu=self.gpu) + compile_cpp(use_mingw=self.mingw, use_gpu=self.gpu, use_mpi=self.mpi, + boost_root=self.boost_root, boost_dir=self.boost_dir, + boost_include_dir=self.boost_include_dir, boost_librarydir=self.boost_librarydir, + opencl_include_dir=self.opencl_include_dir, opencl_library=self.opencl_library) install.run(self) + if os.path.isfile(path_log): + os.remove(path_log) class CustomSdist(sdist): @@ -181,9 +225,11 @@ class CustomSdist(sdist): if __name__ == "__main__": if (8 * struct.calcsize("P")) != 64: - raise Exception('Cannot install LightGBM in 32-bit python, please use 64-bit python instead.') + raise Exception('Cannot install LightGBM in 32-bit Python, please use 64-bit python instead.') dir_path = os.path.dirname(os.path.realpath(__file__)) + path_log = os.path.join(os.path.expanduser('~'), 'LightGBM_compilation.log') + log_notice = "The full version of error log was saved into {0}".format(path_log) if os.path.isfile(os.path.join('..', 'VERSION.txt')): distutils.file_util.copy_file(os.path.join('..', 'VERSION.txt'), os.path.join('.', 'lightgbm'))