From 6bb6164e3c25c603a2b9614b3d51f4ae6ff005e8 Mon Sep 17 00:00:00 2001 From: Nikita Titov Date: Fri, 22 Jan 2021 17:45:43 +0300 Subject: [PATCH] Move compute and eigen libraries to external_libs folder (#3809) * move all submodules to external_libs folder * Update .Rbuildignore * Update MANIFEST.in * Update .appveyor.yml * Update CMakeLists.txt * Update build_r.R * Update test.sh * Update setup.py * Update CMakeLists.txt * Update test.sh * Update setup.py * Update conf.py * Update MANIFEST.in * Update LightGBM.vcxproj * continue * test * test * Update setup.py * hotfix * revert CI tests --- .appveyor.yml | 2 +- .ci/test.sh | 4 +-- .gitmodules | 4 +-- CMakeLists.txt | 4 +-- R-package/.Rbuildignore | 18 +++++------ build-cran-package.sh | 8 ++--- build_r.R | 44 +++++++++++++------------- docs/conf.py | 1 - compute => external_libs/compute | 0 eigen => external_libs/eigen | 0 python-package/MANIFEST.in | 54 ++++++++++++++++---------------- python-package/setup.py | 8 ++--- windows/LightGBM.vcxproj | 14 ++++----- 13 files changed, 81 insertions(+), 80 deletions(-) rename compute => external_libs/compute (100%) rename eigen => external_libs/eigen (100%) diff --git a/.appveyor.yml b/.appveyor.yml index 70bfdf109..96087355b 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -21,7 +21,7 @@ environment: clone_depth: 5 install: - - git submodule update --init --recursive # get `compute` folder + - git submodule update --init --recursive # get `external_libs` folder - set PATH=%PATH:C:\Program Files\Git\usr\bin;=% # delete sh.exe from PATH (mingw32-make fix) - set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH% - set PYTHON_VERSION=%CONFIGURATION% diff --git a/.ci/test.sh b/.ci/test.sh index c4576952b..252b2b49e 100755 --- a/.ci/test.sh +++ b/.ci/test.sh @@ -50,8 +50,8 @@ if [[ $TASK == "lint" ]]; then "r-lintr>=2.0" pip install --user cpplint echo "Linting Python code" - pycodestyle --ignore=E501,W503 --exclude=./compute,./eigen,./.nuget,./external_libs . || exit -1 - pydocstyle --convention=numpy --add-ignore=D105 --match-dir="^(?!^compute|^eigen|external_libs|test|example).*" --match="(?!^test_|setup).*\.py" . || exit -1 + pycodestyle --ignore=E501,W503 --exclude=./.nuget,./external_libs . || exit -1 + pydocstyle --convention=numpy --add-ignore=D105 --match-dir="^(?!^external_libs|test|example).*" --match="(?!^test_|setup).*\.py" . || exit -1 echo "Linting R code" Rscript ${BUILD_DIRECTORY}/.ci/lint_r_code.R ${BUILD_DIRECTORY} || exit -1 echo "Linting C++ code" diff --git a/.gitmodules b/.gitmodules index ccab67d4a..687fa4db9 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,8 +1,8 @@ [submodule "include/boost/compute"] - path = compute + path = external_libs/compute url = https://github.com/boostorg/compute [submodule "eigen"] - path = eigen + path = external_libs/eigen url = https://gitlab.com/libeigen/eigen.git [submodule "external_libs/fmt"] path = external_libs/fmt diff --git a/CMakeLists.txt b/CMakeLists.txt index 8577edb4a..0eda63051 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -83,7 +83,7 @@ if(USE_SWIG) endif() endif(USE_SWIG) -SET(EIGEN_DIR "${PROJECT_SOURCE_DIR}/eigen") +SET(EIGEN_DIR "${PROJECT_SOURCE_DIR}/external_libs/eigen") include_directories(${EIGEN_DIR}) # See https://gitlab.com/libeigen/eigen/-/blob/master/COPYING.README @@ -126,7 +126,7 @@ if(USE_OPENMP) endif(USE_OPENMP) if(USE_GPU) - SET(BOOST_COMPUTE_HEADER_DIR ${PROJECT_SOURCE_DIR}/compute/include) + SET(BOOST_COMPUTE_HEADER_DIR ${PROJECT_SOURCE_DIR}/external_libs/compute/include) include_directories(${BOOST_COMPUTE_HEADER_DIR}) find_package(OpenCL REQUIRED) include_directories(${OpenCL_INCLUDE_DIRS}) diff --git a/R-package/.Rbuildignore b/R-package/.Rbuildignore index 55e51dffc..3e552132c 100644 --- a/R-package/.Rbuildignore +++ b/R-package/.Rbuildignore @@ -19,15 +19,15 @@ AUTOCONF_UBUNTU_VERSION ^.*\.so ^src/build/.*$ ^src/CMakeLists.txt$ -^src/compute/.appveyor.yml$ -^src/compute/.coveralls.yml$ -^src/compute/.travis.yml$ -^src/compute/test/.*$ -^src/compute/index.html$ -^src/compute/.git$ -^src/compute/.gitignore$ -^src/compute/CONTRIBUTING.md$ -^src/compute/README.md$ +^src/external_libs/compute/.appveyor.yml$ +^src/external_libs/compute/.coveralls.yml$ +^src/external_libs/compute/.travis.yml$ +^src/external_libs/compute/test/.*$ +^src/external_libs/compute/index.html$ +^src/external_libs/compute/.git$ +^src/external_libs/compute/.gitignore$ +^src/external_libs/compute/CONTRIBUTING.md$ +^src/external_libs/compute/README.md$ src/external_libs/fast_double_parser/benchmarks src/external_libs/fast_double_parser/Makefile src/external_libs/fast_double_parser/.*\.md diff --git a/build-cran-package.sh b/build-cran-package.sh index c4c40cba4..72e61f7c1 100755 --- a/build-cran-package.sh +++ b/build-cran-package.sh @@ -45,18 +45,18 @@ mkdir -p ${EIGEN_R_DIR} modules="Cholesky Core Dense Eigenvalues Geometry Householder Jacobi LU QR SVD" for eigen_module in ${modules}; do - cp eigen/Eigen/${eigen_module} ${EIGEN_R_DIR}/${eigen_module} + cp external_libs/eigen/Eigen/${eigen_module} ${EIGEN_R_DIR}/${eigen_module} if [ ${eigen_module} != "Dense" ]; then mkdir -p ${EIGEN_R_DIR}/src/${eigen_module}/ - cp -R eigen/Eigen/src/${eigen_module}/* ${EIGEN_R_DIR}/src/${eigen_module}/ + cp -R external_libs/eigen/Eigen/src/${eigen_module}/* ${EIGEN_R_DIR}/src/${eigen_module}/ fi done mkdir -p ${EIGEN_R_DIR}/src/misc -cp -R eigen/Eigen/src/misc/* ${EIGEN_R_DIR}/src/misc/ +cp -R external_libs/eigen/Eigen/src/misc/* ${EIGEN_R_DIR}/src/misc/ mkdir -p ${EIGEN_R_DIR}/src/plugins -cp -R eigen/Eigen/src/plugins/* ${EIGEN_R_DIR}/src/plugins/ +cp -R external_libs/eigen/Eigen/src/plugins/* ${EIGEN_R_DIR}/src/plugins/ cd ${TEMP_R_DIR} diff --git a/build_r.R b/build_r.R index 0bbe42fc8..d96447c2e 100644 --- a/build_r.R +++ b/build_r.R @@ -206,18 +206,6 @@ result <- file.copy( ) .handle_result(result) -# compute/ is a submodule with boost, only needed if -# building the R package with GPU support -if (USING_GPU) { - result <- file.copy( - from = "compute/" - , to = sprintf("%s/", TEMP_SOURCE_DIR) - , recursive = TRUE - , overwrite = TRUE - ) - .handle_result(result) -} - EIGEN_R_DIR <- file.path(TEMP_SOURCE_DIR, "include", "Eigen") dir.create(EIGEN_R_DIR) @@ -235,7 +223,7 @@ eigen_modules <- c( ) for (eigen_module in eigen_modules) { result <- file.copy( - from = file.path("eigen", "Eigen", eigen_module) + from = file.path("external_libs", "eigen", "Eigen", eigen_module) , to = EIGEN_R_DIR , recursive = FALSE , overwrite = TRUE @@ -252,7 +240,7 @@ for (eigen_module in c(eigen_modules, "misc", "plugins")) { module_dir <- file.path(EIGEN_R_DIR, "src", eigen_module) dir.create(module_dir, recursive = TRUE) result <- file.copy( - from = sprintf("%s/", file.path("eigen", "Eigen", "src", eigen_module)) + from = sprintf("%s/", file.path("external_libs", "eigen", "Eigen", "src", eigen_module)) , to = sprintf("%s/", file.path(EIGEN_R_DIR, "src")) , recursive = TRUE , overwrite = TRUE @@ -305,13 +293,27 @@ result <- file.remove( #------------# # submodules # #------------# -result <- file.copy( - from = "external_libs/" - , to = sprintf("%s/", TEMP_SOURCE_DIR) - , recursive = TRUE - , overwrite = TRUE -) -.handle_result(result) +EXTERNAL_LIBS_R_DIR <- file.path(TEMP_SOURCE_DIR, "external_libs") +dir.create(EXTERNAL_LIBS_R_DIR) +for (submodule in list.dirs( + path = "external_libs" + , full.names = FALSE + , recursive = FALSE +)) { + # compute/ is a submodule with boost, only needed if + # building the R package with GPU support; + # eigen/ has a special treatment due to licensing aspects + if ((submodule == "compute" && !USING_GPU) || submodule == "eigen") { + next + } + result <- file.copy( + from = sprintf("%s/", file.path("external_libs", submodule)) + , to = sprintf("%s/", EXTERNAL_LIBS_R_DIR) + , recursive = TRUE + , overwrite = TRUE + ) + .handle_result(result) +} # copy files into the place CMake expects for (src_file in c("lightgbm_R.cpp", "lightgbm_R.h", "R_object_helper.h")) { diff --git a/docs/conf.py b/docs/conf.py index d1a51a0a6..b84321b95 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -220,7 +220,6 @@ def generate_doxygen_xml(app): "SKIP_FUNCTION_MACROS=NO", "SORT_BRIEF_DOCS=YES", "WARN_AS_ERROR=YES", - "EXCLUDE_PATTERNS=*/eigen/*" ] doxygen_input = '\n'.join(doxygen_args) doxygen_input = bytes(doxygen_input, "utf-8") diff --git a/compute b/external_libs/compute similarity index 100% rename from compute rename to external_libs/compute diff --git a/eigen b/external_libs/eigen similarity index 100% rename from eigen rename to external_libs/eigen diff --git a/python-package/MANIFEST.in b/python-package/MANIFEST.in index 1f5f58510..7fad0fa42 100644 --- a/python-package/MANIFEST.in +++ b/python-package/MANIFEST.in @@ -6,32 +6,32 @@ include compile/CMakeLists.txt include compile/CMakeIntegratedOpenCL.cmake recursive-include compile *.so recursive-include compile/Release *.dll -include compile/compute/CMakeLists.txt -recursive-include compile/compute/cmake * -recursive-include compile/compute/include * -recursive-include compile/compute/meta * -include compile/eigen/CMakeLists.txt -include compile/eigen/Eigen/Cholesky -include compile/eigen/Eigen/Core -include compile/eigen/Eigen/Dense -include compile/eigen/Eigen/Eigenvalues -include compile/eigen/Eigen/Geometry -include compile/eigen/Eigen/Householder -include compile/eigen/Eigen/Jacobi -include compile/eigen/Eigen/LU -include compile/eigen/Eigen/QR -include compile/eigen/Eigen/SVD -recursive-include compile/eigen/Eigen/src/Cholesky * -recursive-include compile/eigen/Eigen/src/Core * -recursive-include compile/eigen/Eigen/src/Eigenvalues * -recursive-include compile/eigen/Eigen/src/Geometry * -recursive-include compile/eigen/Eigen/src/Householder * -recursive-include compile/eigen/Eigen/src/Jacobi * -recursive-include compile/eigen/Eigen/src/LU * -recursive-include compile/eigen/Eigen/src/misc * -recursive-include compile/eigen/Eigen/src/plugins * -recursive-include compile/eigen/Eigen/src/QR * -recursive-include compile/eigen/Eigen/src/SVD * +include compile/external_libs/compute/CMakeLists.txt +recursive-include compile/external_libs/compute/cmake * +recursive-include compile/external_libs/compute/include * +recursive-include compile/external_libs/compute/meta * +include compile/external_libs/eigen/CMakeLists.txt +include compile/external_libs/eigen/Eigen/Cholesky +include compile/external_libs/eigen/Eigen/Core +include compile/external_libs/eigen/Eigen/Dense +include compile/external_libs/eigen/Eigen/Eigenvalues +include compile/external_libs/eigen/Eigen/Geometry +include compile/external_libs/eigen/Eigen/Householder +include compile/external_libs/eigen/Eigen/Jacobi +include compile/external_libs/eigen/Eigen/LU +include compile/external_libs/eigen/Eigen/QR +include compile/external_libs/eigen/Eigen/SVD +recursive-include compile/external_libs/eigen/Eigen/src/Cholesky * +recursive-include compile/external_libs/eigen/Eigen/src/Core * +recursive-include compile/external_libs/eigen/Eigen/src/Eigenvalues * +recursive-include compile/external_libs/eigen/Eigen/src/Geometry * +recursive-include compile/external_libs/eigen/Eigen/src/Householder * +recursive-include compile/external_libs/eigen/Eigen/src/Jacobi * +recursive-include compile/external_libs/eigen/Eigen/src/LU * +recursive-include compile/external_libs/eigen/Eigen/src/misc * +recursive-include compile/external_libs/eigen/Eigen/src/plugins * +recursive-include compile/external_libs/eigen/Eigen/src/QR * +recursive-include compile/external_libs/eigen/Eigen/src/SVD * include compile/external_libs/fast_double_parser/CMakeLists.txt include compile/external_libs/fast_double_parser/LICENSE include compile/external_libs/fast_double_parser/LICENSE.BSL @@ -44,4 +44,4 @@ recursive-include compile/src * recursive-include compile/windows LightGBM.sln LightGBM.vcxproj recursive-include compile/windows/x64/DLL *.dll global-exclude *.py[co] -exclude compile/compute/.git +exclude compile/external_libs/compute/.git diff --git a/python-package/setup.py b/python-package/setup.py index 072a5fce3..3e545c490 100644 --- a/python-package/setup.py +++ b/python-package/setup.py @@ -65,8 +65,10 @@ def copy_files(integrated_opencl=False, use_gpu=False): if not os.path.isfile(os.path.join(CURRENT_DIR, '_IS_SOURCE_PACKAGE.txt')): copy_files_helper('include') copy_files_helper('src') - copy_files_helper('eigen') - copy_files_helper('external_libs') + for submodule in os.listdir(os.path.join(CURRENT_DIR, os.path.pardir, 'external_libs')): + if submodule == 'compute' and not use_gpu: + continue + copy_files_helper(os.path.join('external_libs', submodule)) if not os.path.exists(os.path.join(CURRENT_DIR, "compile", "windows")): os.makedirs(os.path.join(CURRENT_DIR, "compile", "windows")) copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "windows", "LightGBM.sln"), @@ -85,8 +87,6 @@ def copy_files(integrated_opencl=False, use_gpu=False): copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "CMakeIntegratedOpenCL.cmake"), os.path.join(CURRENT_DIR, "compile", "CMakeIntegratedOpenCL.cmake"), verbose=0) - if use_gpu: - copy_files_helper('compute') def clear_path(path): diff --git a/windows/LightGBM.vcxproj b/windows/LightGBM.vcxproj index 5b9190fb2..59b589a40 100644 --- a/windows/LightGBM.vcxproj +++ b/windows/LightGBM.vcxproj @@ -117,7 +117,7 @@ Disabled MultiThreadedDebugDLL true - $(ProjectDir)\..\eigen;%(AdditionalIncludeDirectories) + $(ProjectDir)\..\external_libs\eigen;%(AdditionalIncludeDirectories) @@ -140,7 +140,7 @@ Disabled MultiThreadedDebugDLL true - $(ProjectDir)\..\eigen;%(AdditionalIncludeDirectories) + $(ProjectDir)\..\external_libs\eigen;%(AdditionalIncludeDirectories) @@ -160,7 +160,7 @@ Disabled MultiThreadedDebugDLL true - $(ProjectDir)\..\eigen;%(AdditionalIncludeDirectories) + $(ProjectDir)\..\external_libs\eigen;%(AdditionalIncludeDirectories) @@ -183,7 +183,7 @@ true MultiThreadedDLL true - $(ProjectDir)\..\eigen;%(AdditionalIncludeDirectories) + $(ProjectDir)\..\external_libs\eigen;%(AdditionalIncludeDirectories) @@ -210,7 +210,7 @@ true true true - $(ProjectDir)\..\eigen;%(AdditionalIncludeDirectories) + $(ProjectDir)\..\external_libs\eigen;%(AdditionalIncludeDirectories) @@ -231,7 +231,7 @@ true true true - $(ProjectDir)\..\eigen;%(AdditionalIncludeDirectories) + $(ProjectDir)\..\external_libs\eigen;%(AdditionalIncludeDirectories) @@ -341,4 +341,4 @@ - \ No newline at end of file +