зеркало из https://github.com/microsoft/LightGBM.git
Add option to build with integrated OpenCL (#3144)
* Add specialized OpenCL/Python package build path * Refer to upstream OpenCL repository * Reset build job count in setup.py * TEMPORARY: refer to OpenCL fork to ensure Linux CI builds succeed * Remove intermediate cmake target * Restrict OpenCL headers to documented API version * Use command line definition to activate integrated build * Flag reference to unofficial repo with FIXME * TEMPORARY: update private repo tag for dependency * Remove integrated build for non-Win32 and related cleanup * Remove commented code * Rename integrated OpenCL build option and other cleanups * Small cleanups * Update CMakeIntegratedOpenCL.cmake Co-authored-by: Nikita Titov <nekit94-08@mail.ru> * Update CMakeIntegratedOpenCL.cmake Co-authored-by: Nikita Titov <nekit94-08@mail.ru> * Update CMakeIntegratedOpenCL.cmake Co-authored-by: Nikita Titov <nekit94-08@mail.ru> * Update CMakeIntegratedOpenCL.cmake Co-authored-by: Nikita Titov <nekit94-08@mail.ru> * Update CMakeLists.txt Co-authored-by: Nikita Titov <nekit94-08@mail.ru> * Update CMakeLists.txt Co-authored-by: Nikita Titov <nekit94-08@mail.ru> * Update CMakeLists.txt Co-authored-by: Nikita Titov <nekit94-08@mail.ru> * Update CMakeIntegratedOpenCL.cmake Targeted download of Boost submodules. Co-authored-by: Nikita Titov <nekit94-08@mail.ru> * Update CMakeLists.txt Co-authored-by: Nikita Titov <nekit94-08@mail.ru> Co-authored-by: Nikita Titov <nekit94-08@mail.ru>
This commit is contained in:
TP Boudreau
GitHub
Родитель
eff287e9e1
Коммит
3454698efe
|
|
@ -0,0 +1,85 @@
|
|||
set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE)
|
||||
set(BOOST_VERSION_DOT "1.74")
|
||||
string(REPLACE "." "_" BOOST_VERSION_UNDERSCORE ${BOOST_VERSION_DOT})
|
||||
|
||||
set(OPENCL_HEADER_REPOSITORY "https://github.com/KhronosGroup/OpenCL-Headers.git")
|
||||
set(OPENCL_HEADER_TAG "1b2a1850f410aaaaeaa56cead5a179b5aea4918e")
|
||||
|
||||
set(OPENCL_LOADER_REPOSITORY "https://github.com/KhronosGroup/OpenCL-ICD-Loader.git")
|
||||
set(OPENCL_LOADER_TAG "98ca71fb9f8484f1cd1999f55224bf9e8d18693b")
|
||||
|
||||
set(BOOST_REPOSITORY "https://github.com/boostorg/boost.git")
|
||||
set(BOOST_TAG "boost-${BOOST_VERSION_DOT}.0")
|
||||
|
||||
# Build Independent OpenCL library
|
||||
include(FetchContent)
|
||||
FetchContent_Declare(OpenCL-Headers GIT_REPOSITORY ${OPENCL_HEADER_REPOSITORY} GIT_TAG ${OPENCL_HEADER_TAG})
|
||||
FetchContent_GetProperties(OpenCL-Headers)
|
||||
if(NOT OpenCL-Headers_POPULATED)
|
||||
FetchContent_Populate(OpenCL-Headers)
|
||||
message(STATUS "Populated OpenCL Headers")
|
||||
endif()
|
||||
set(OPENCL_ICD_LOADER_HEADERS_DIR ${opencl-headers_SOURCE_DIR} CACHE PATH "") # for OpenCL ICD Loader
|
||||
set(OpenCL_INCLUDE_DIR ${opencl-headers_SOURCE_DIR} CACHE PATH "") # for Boost::Compute
|
||||
|
||||
FetchContent_Declare(OpenCL-ICD-Loader GIT_REPOSITORY ${OPENCL_LOADER_REPOSITORY} GIT_TAG ${OPENCL_LOADER_TAG})
|
||||
FetchContent_GetProperties(OpenCL-ICD-Loader)
|
||||
if(NOT OpenCL-ICD-Loader_POPULATED)
|
||||
FetchContent_Populate(OpenCL-ICD-Loader)
|
||||
set(USE_DYNAMIC_VCXX_RUNTIME ON)
|
||||
add_subdirectory(${opencl-icd-loader_SOURCE_DIR} ${opencl-icd-loader_BINARY_DIR} EXCLUDE_FROM_ALL)
|
||||
message(STATUS "Populated OpenCL ICD Loader")
|
||||
endif()
|
||||
list(APPEND INTEGRATED_OPENCL_INCLUDES ${OPENCL_ICD_LOADER_HEADERS_DIR})
|
||||
list(APPEND INTEGRATED_OPENCL_LIBRARIES ${opencl-icd-loader_BINARY_DIR}/Release/OpenCL.lib cfgmgr32.lib runtimeobject.lib)
|
||||
list(APPEND INTEGRATED_OPENCL_DEFINITIONS CL_TARGET_OPENCL_VERSION=120)
|
||||
|
||||
# Build Independent Boost libraries
|
||||
include(ExternalProject)
|
||||
include(ProcessorCount)
|
||||
ProcessorCount(J)
|
||||
set(BOOST_BASE "${PROJECT_BINARY_DIR}/Boost")
|
||||
set(BOOST_BOOTSTRAP "${BOOST_BASE}/source/bootstrap.bat")
|
||||
set(BOOST_BUILD "${BOOST_BASE}/source/b2.exe")
|
||||
set(BOOST_FLAGS "")
|
||||
list(APPEND BOOST_SUBMODULES "libs/algorithm" "libs/align" "libs/any" "libs/array" "libs/assert" "libs/bind" "libs/chrono" "libs/compute" "libs/concept_check" "libs/config" "libs/container" "libs/container_hash" "libs/core" "libs/detail" "libs/filesystem" "libs/foreach" "libs/format" "libs/function" "libs/function_types" "libs/fusion" "libs/headers" "libs/integer" "libs/io" "libs/iterator" "libs/lexical_cast" "libs/math" "libs/move" "libs/mpl" "libs/multi_index" "libs/numeric/conversion" "libs/optional" "libs/predef" "libs/preprocessor" "libs/property_tree" "libs/range" "libs/ratio" "libs/serialization" "libs/smart_ptr" "libs/static_assert" "libs/system" "libs/throw_exception" "libs/tuple" "libs/typeof" "libs/type_index" "libs/type_traits" "libs/utility" "libs/uuid" "libs/winapi" "tools/boost_install" "tools/build")
|
||||
ExternalProject_Add(Boost
|
||||
TMP_DIR "${BOOST_BASE}/tmp"
|
||||
STAMP_DIR "${BOOST_BASE}/stamp"
|
||||
DOWNLOAD_DIR "${BOOST_BASE}/download"
|
||||
SOURCE_DIR "${BOOST_BASE}/source"
|
||||
BINARY_DIR "${BOOST_BASE}/source"
|
||||
INSTALL_DIR "${BOOST_BASE}/install"
|
||||
GIT_REPOSITORY ${BOOST_REPOSITORY}
|
||||
GIT_TAG ${BOOST_TAG}
|
||||
GIT_SUBMODULES ${BOOST_SUBMODULES}
|
||||
GIT_SHALLOW ON
|
||||
UPDATE_COMMAND ""
|
||||
PATCH_COMMAND ""
|
||||
CONFIGURE_COMMAND ${BOOST_BOOTSTRAP}
|
||||
BUILD_COMMAND ${BOOST_BUILD} -sBOOST_ROOT=${BOOST_BASE}/source -a -q -j ${J} --with-headers --with-chrono --with-filesystem --with-system link=static runtime-link=shared variant=release threading=multi cxxflags="${BOOST_FLAGS}"
|
||||
INSTALL_COMMAND ""
|
||||
)
|
||||
set(BOOST_INCLUDE "${BOOST_BASE}/source" CACHE PATH "")
|
||||
set(BOOST_LIBRARY "${BOOST_BASE}/source/stage/lib" CACHE PATH "")
|
||||
list(APPEND INTEGRATED_OPENCL_INCLUDES ${BOOST_INCLUDE})
|
||||
if(MSVC)
|
||||
if(${MSVC_VERSION} GREATER 1929)
|
||||
message(FATAL_ERROR "Unrecognized MSVC version number: ${MSVC_VERSION}")
|
||||
elseif(${MSVC_VERSION} GREATER 1919)
|
||||
set(MSVC_TOOLCHAIN_ID "142")
|
||||
elseif(${MSVC_VERSION} GREATER 1909)
|
||||
set(MSVC_TOOLCHAIN_ID "141")
|
||||
elseif(${MSVC_VERSION} GREATER 1899)
|
||||
set(MSVC_TOOLCHAIN_ID "140")
|
||||
else()
|
||||
message(FATAL_ERROR "Unrecognized MSVC version number: ${MSVC_VERSION}")
|
||||
endif()
|
||||
list(APPEND INTEGRATED_OPENCL_LIBRARIES ${BOOST_LIBRARY}/libboost_filesystem-vc${MSVC_TOOLCHAIN_ID}-mt-x64-${BOOST_VERSION_UNDERSCORE}.lib)
|
||||
list(APPEND INTEGRATED_OPENCL_LIBRARIES ${BOOST_LIBRARY}/libboost_system-vc${MSVC_TOOLCHAIN_ID}-mt-x64-${BOOST_VERSION_UNDERSCORE}.lib)
|
||||
list(APPEND INTEGRATED_OPENCL_LIBRARIES ${BOOST_LIBRARY}/libboost_chrono-vc${MSVC_TOOLCHAIN_ID}-mt-x64-${BOOST_VERSION_UNDERSCORE}.lib)
|
||||
else()
|
||||
message(FATAL_ERROR "Integrated OpenCL build is not yet available for MinGW")
|
||||
endif()
|
||||
|
||||
set(BUILD_SHARED_LIBS ON CACHE BOOL "" FORCE)
|
||||
|
|
@ -1,4 +1,6 @@
|
|||
if(USE_GPU OR APPLE)
|
||||
if(__INTEGRATE_OPENCL)
|
||||
cmake_minimum_required(VERSION 3.11)
|
||||
elseif(USE_GPU OR APPLE)
|
||||
cmake_minimum_required(VERSION 3.2)
|
||||
elseif(USE_CUDA)
|
||||
cmake_minimum_required(VERSION 3.16)
|
||||
|
|
@ -22,11 +24,18 @@ OPTION(USE_CUDA "Enable CUDA-accelerated training (EXPERIMENTAL)" OFF)
|
|||
OPTION(USE_DEBUG "Set to ON for Debug mode" OFF)
|
||||
OPTION(BUILD_STATIC_LIB "Build static library" OFF)
|
||||
OPTION(__BUILD_FOR_R "Set to ON if building lib_lightgbm for use with the R package" OFF)
|
||||
OPTION(__INTEGRATE_OPENCL "Set to ON if building LightGBM with the OpenCL ICD Loader and its dependencies included" OFF)
|
||||
|
||||
if(APPLE)
|
||||
OPTION(APPLE_OUTPUT_DYLIB "Output dylib shared library" OFF)
|
||||
endif(APPLE)
|
||||
|
||||
if(__INTEGRATE_OPENCL)
|
||||
set(__INTEGRATE_OPENCL ON CACHE BOOL "" FORCE)
|
||||
set(USE_GPU OFF CACHE BOOL "" FORCE)
|
||||
message(STATUS "Building library with integrated OpenCL components")
|
||||
endif()
|
||||
|
||||
if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
|
||||
if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "4.8.2")
|
||||
message(FATAL_ERROR "Insufficient gcc version")
|
||||
|
|
@ -134,6 +143,15 @@ if(USE_GPU)
|
|||
ADD_DEFINITIONS(-DUSE_GPU)
|
||||
endif(USE_GPU)
|
||||
|
||||
if(__INTEGRATE_OPENCL)
|
||||
if(WIN32)
|
||||
include(CMakeIntegratedOpenCL.cmake)
|
||||
ADD_DEFINITIONS(-DUSE_GPU)
|
||||
else()
|
||||
message(FATAL_ERROR "Integrated OpenCL build is available only for Windows")
|
||||
endif(WIN32)
|
||||
endif(__INTEGRATE_OPENCL)
|
||||
|
||||
if(USE_CUDA)
|
||||
find_package(CUDA REQUIRED)
|
||||
include_directories(${CUDA_INCLUDE_DIRS})
|
||||
|
|
@ -321,7 +339,7 @@ else()
|
|||
endif(BUILD_STATIC_LIB)
|
||||
|
||||
if(MSVC)
|
||||
set_target_properties(_lightgbm PROPERTIES OUTPUT_NAME "lib_lightgbm")
|
||||
set_target_properties(_lightgbm PROPERTIES OUTPUT_NAME "lib_lightgbm")
|
||||
endif(MSVC)
|
||||
|
||||
if(USE_SWIG)
|
||||
|
|
@ -378,6 +396,19 @@ if(USE_GPU)
|
|||
TARGET_LINK_LIBRARIES(_lightgbm ${OpenCL_LIBRARY} ${Boost_LIBRARIES})
|
||||
endif(USE_GPU)
|
||||
|
||||
if(__INTEGRATE_OPENCL)
|
||||
# targets OpenCL and Boost are added in CMakeIntegratedOpenCL.cmake
|
||||
add_dependencies(lightgbm OpenCL Boost)
|
||||
add_dependencies(_lightgbm OpenCL Boost)
|
||||
# variables INTEGRATED_OPENCL_* are set in CMakeIntegratedOpenCL.cmake
|
||||
target_include_directories(lightgbm PRIVATE ${INTEGRATED_OPENCL_INCLUDES})
|
||||
target_include_directories(_lightgbm PRIVATE ${INTEGRATED_OPENCL_INCLUDES})
|
||||
target_compile_definitions(lightgbm PRIVATE ${INTEGRATED_OPENCL_DEFINITIONS})
|
||||
target_compile_definitions(_lightgbm PRIVATE ${INTEGRATED_OPENCL_DEFINITIONS})
|
||||
target_link_libraries(lightgbm PRIVATE ${INTEGRATED_OPENCL_LIBRARIES})
|
||||
target_link_libraries(_lightgbm PRIVATE ${INTEGRATED_OPENCL_LIBRARIES})
|
||||
endif()
|
||||
|
||||
if(USE_CUDA)
|
||||
set_target_properties(lightgbm PROPERTIES CUDA_RESOLVE_DEVICE_SYMBOLS ON)
|
||||
TARGET_LINK_LIBRARIES(
|
||||
|
|
@ -396,12 +427,14 @@ if(USE_HDFS)
|
|||
TARGET_LINK_LIBRARIES(_lightgbm ${HDFS_CXX_LIBRARIES})
|
||||
endif(USE_HDFS)
|
||||
|
||||
if(WIN32 AND (MINGW OR CYGWIN))
|
||||
TARGET_LINK_LIBRARIES(lightgbm Ws2_32)
|
||||
TARGET_LINK_LIBRARIES(_lightgbm Ws2_32)
|
||||
TARGET_LINK_LIBRARIES(lightgbm IPHLPAPI)
|
||||
TARGET_LINK_LIBRARIES(_lightgbm IPHLPAPI)
|
||||
endif()
|
||||
if(WIN32)
|
||||
if(MINGW OR CYGWIN)
|
||||
TARGET_LINK_LIBRARIES(lightgbm Ws2_32)
|
||||
TARGET_LINK_LIBRARIES(_lightgbm Ws2_32)
|
||||
TARGET_LINK_LIBRARIES(lightgbm IPHLPAPI)
|
||||
TARGET_LINK_LIBRARIES(_lightgbm IPHLPAPI)
|
||||
endif(MINGW OR CYGWIN)
|
||||
endif(WIN32)
|
||||
|
||||
if(__BUILD_FOR_R)
|
||||
if(MSVC)
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ def find_lib():
|
|||
return LIB_PATH
|
||||
|
||||
|
||||
def copy_files(use_gpu=False):
|
||||
def copy_files(integrated_opencl=False, use_gpu=False):
|
||||
|
||||
def copy_files_helper(folder_name):
|
||||
src = os.path.join(CURRENT_DIR, os.path.pardir, folder_name)
|
||||
|
|
@ -51,14 +51,18 @@ def copy_files(use_gpu=False):
|
|||
copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "windows", "LightGBM.vcxproj"),
|
||||
os.path.join(CURRENT_DIR, "compile", "windows", "LightGBM.vcxproj"),
|
||||
verbose=0)
|
||||
if use_gpu:
|
||||
copy_files_helper('compute')
|
||||
copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "CMakeLists.txt"),
|
||||
os.path.join(CURRENT_DIR, "compile", "CMakeLists.txt"),
|
||||
verbose=0)
|
||||
copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "LICENSE"),
|
||||
os.path.join(CURRENT_DIR, "LICENSE"),
|
||||
verbose=0)
|
||||
copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "CMakeLists.txt"),
|
||||
os.path.join(CURRENT_DIR, "compile", "CMakeLists.txt"),
|
||||
verbose=0)
|
||||
if integrated_opencl:
|
||||
copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "CMakeIntegratedOpenCL.cmake"),
|
||||
os.path.join(CURRENT_DIR, "compile", "CMakeIntegratedOpenCL.cmake"),
|
||||
verbose=0)
|
||||
if use_gpu:
|
||||
copy_files_helper('compute')
|
||||
|
||||
|
||||
def clear_path(path):
|
||||
|
|
@ -91,7 +95,7 @@ def compile_cpp(use_mingw=False, use_gpu=False, use_cuda=False, use_mpi=False,
|
|||
use_hdfs=False, boost_root=None, boost_dir=None,
|
||||
boost_include_dir=None, boost_librarydir=None,
|
||||
opencl_include_dir=None, opencl_library=None,
|
||||
nomp=False, bit32=False):
|
||||
nomp=False, bit32=False, integrated_opencl=False):
|
||||
|
||||
if os.path.exists(os.path.join(CURRENT_DIR, "build_cpp")):
|
||||
shutil.rmtree(os.path.join(CURRENT_DIR, "build_cpp"))
|
||||
|
|
@ -101,6 +105,9 @@ def compile_cpp(use_mingw=False, use_gpu=False, use_cuda=False, use_mpi=False,
|
|||
logger.info("Starting to compile the library.")
|
||||
|
||||
cmake_cmd = ["cmake", "../compile/"]
|
||||
if integrated_opencl:
|
||||
use_gpu = False
|
||||
cmake_cmd.append("-D__INTEGRATE_OPENCL=ON")
|
||||
if use_gpu:
|
||||
cmake_cmd.append("-DUSE_GPU=ON")
|
||||
if boost_root:
|
||||
|
|
@ -136,7 +143,7 @@ def compile_cpp(use_mingw=False, use_gpu=False, use_cuda=False, use_mpi=False,
|
|||
else:
|
||||
status = 1
|
||||
lib_path = os.path.join(CURRENT_DIR, "compile", "windows", "x64", "DLL", "lib_lightgbm.dll")
|
||||
if not any((use_gpu, use_mpi, use_hdfs, nomp, bit32)):
|
||||
if not any((use_gpu, use_mpi, use_hdfs, nomp, bit32, integrated_opencl)):
|
||||
logger.info("Starting to compile with MSBuild from existing solution file.")
|
||||
platform_toolsets = ("v142", "v141", "v140")
|
||||
for pt in platform_toolsets:
|
||||
|
|
@ -189,6 +196,7 @@ class CustomInstall(install):
|
|||
|
||||
user_options = install.user_options + [
|
||||
('mingw', 'm', 'Compile with MinGW'),
|
||||
('integrated-opencl', None, 'Compile integrated OpenCL version'),
|
||||
('gpu', 'g', 'Compile GPU version'),
|
||||
('cuda', None, 'Compile CUDA version'),
|
||||
('mpi', None, 'Compile MPI version'),
|
||||
|
|
@ -207,6 +215,7 @@ class CustomInstall(install):
|
|||
def initialize_options(self):
|
||||
install.initialize_options(self)
|
||||
self.mingw = 0
|
||||
self.integrated_opencl = 0
|
||||
self.gpu = 0
|
||||
self.cuda = 0
|
||||
self.boost_root = None
|
||||
|
|
@ -231,12 +240,12 @@ class CustomInstall(install):
|
|||
"please use 64-bit Python instead.")
|
||||
open(LOG_PATH, 'wb').close()
|
||||
if not self.precompile:
|
||||
copy_files(use_gpu=self.gpu)
|
||||
copy_files(integrated_opencl=self.integrated_opencl, use_gpu=self.gpu)
|
||||
compile_cpp(use_mingw=self.mingw, use_gpu=self.gpu, use_cuda=self.cuda, use_mpi=self.mpi,
|
||||
use_hdfs=self.hdfs, boost_root=self.boost_root, boost_dir=self.boost_dir,
|
||||
boost_include_dir=self.boost_include_dir, boost_librarydir=self.boost_librarydir,
|
||||
opencl_include_dir=self.opencl_include_dir, opencl_library=self.opencl_library,
|
||||
nomp=self.nomp, bit32=self.bit32)
|
||||
nomp=self.nomp, bit32=self.bit32, integrated_opencl=self.integrated_opencl)
|
||||
install.run(self)
|
||||
if os.path.isfile(LOG_PATH):
|
||||
os.remove(LOG_PATH)
|
||||
|
|
@ -245,7 +254,7 @@ class CustomInstall(install):
|
|||
class CustomSdist(sdist):
|
||||
|
||||
def run(self):
|
||||
copy_files(use_gpu=True)
|
||||
copy_files(integrated_opencl=True, use_gpu=True)
|
||||
open(os.path.join(CURRENT_DIR, '_IS_SOURCE_PACKAGE.txt'), 'w').close()
|
||||
if os.path.exists(os.path.join(CURRENT_DIR, 'lightgbm', 'Release')):
|
||||
shutil.rmtree(os.path.join(CURRENT_DIR, 'lightgbm', 'Release'))
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче