Move compute and eigen libraries to external_libs folder (#3809)

* move all submodules to external_libs folder

* Update .Rbuildignore

* Update MANIFEST.in

* Update .appveyor.yml

* Update CMakeLists.txt

* Update build_r.R

* Update test.sh

* Update setup.py

* Update CMakeLists.txt

* Update test.sh

* Update setup.py

* Update conf.py

* Update MANIFEST.in

* Update LightGBM.vcxproj

* continue

* test

* test

* Update setup.py

* hotfix

* revert CI tests

.appveyor.yml

@@ -21,7 +21,7 @@ environment:
 clone_depth: 5
 
 install:
-  - git submodule update --init --recursive  # get `compute` folder
+  - git submodule update --init --recursive  # get `external_libs` folder
   - set PATH=%PATH:C:\Program Files\Git\usr\bin;=%  # delete sh.exe from PATH (mingw32-make fix)
   - set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH%
   - set PYTHON_VERSION=%CONFIGURATION%

.ci/test.sh

@@ -50,8 +50,8 @@ if [[ $TASK == "lint" ]]; then
             "r-lintr>=2.0"
     pip install --user cpplint
     echo "Linting Python code"
-    pycodestyle --ignore=E501,W503 --exclude=./compute,./eigen,./.nuget,./external_libs . || exit -1
-    pydocstyle --convention=numpy --add-ignore=D105 --match-dir="^(?!^compute|^eigen|external_libs|test|example).*" --match="(?!^test_|setup).*\.py" . || exit -1
+    pycodestyle --ignore=E501,W503 --exclude=./.nuget,./external_libs . || exit -1
+    pydocstyle --convention=numpy --add-ignore=D105 --match-dir="^(?!^external_libs|test|example).*" --match="(?!^test_|setup).*\.py" . || exit -1
     echo "Linting R code"
     Rscript ${BUILD_DIRECTORY}/.ci/lint_r_code.R ${BUILD_DIRECTORY} || exit -1
     echo "Linting C++ code"

.gitmodules

@@ -1,8 +1,8 @@
 [submodule "include/boost/compute"]
-	path = compute
+	path = external_libs/compute
 	url = https://github.com/boostorg/compute
 [submodule "eigen"]
-	path = eigen
+	path = external_libs/eigen
 	url = https://gitlab.com/libeigen/eigen.git
 [submodule "external_libs/fmt"]
 	path = external_libs/fmt

CMakeLists.txt

@@ -83,7 +83,7 @@ if(USE_SWIG)
   endif()
 endif(USE_SWIG)
 
-SET(EIGEN_DIR "${PROJECT_SOURCE_DIR}/eigen")
+SET(EIGEN_DIR "${PROJECT_SOURCE_DIR}/external_libs/eigen")
 include_directories(${EIGEN_DIR})
 
 # See https://gitlab.com/libeigen/eigen/-/blob/master/COPYING.README
@@ -126,7 +126,7 @@ if(USE_OPENMP)
 endif(USE_OPENMP)
 
 if(USE_GPU)
-    SET(BOOST_COMPUTE_HEADER_DIR ${PROJECT_SOURCE_DIR}/compute/include)
+    SET(BOOST_COMPUTE_HEADER_DIR ${PROJECT_SOURCE_DIR}/external_libs/compute/include)
     include_directories(${BOOST_COMPUTE_HEADER_DIR})
     find_package(OpenCL REQUIRED)
     include_directories(${OpenCL_INCLUDE_DIRS})

R-package/.Rbuildignore

@@ -19,15 +19,15 @@ AUTOCONF_UBUNTU_VERSION
 ^.*\.so
 ^src/build/.*$
 ^src/CMakeLists.txt$
-^src/compute/.appveyor.yml$
-^src/compute/.coveralls.yml$
-^src/compute/.travis.yml$
-^src/compute/test/.*$
-^src/compute/index.html$
-^src/compute/.git$
-^src/compute/.gitignore$
-^src/compute/CONTRIBUTING.md$
-^src/compute/README.md$
+^src/external_libs/compute/.appveyor.yml$
+^src/external_libs/compute/.coveralls.yml$
+^src/external_libs/compute/.travis.yml$
+^src/external_libs/compute/test/.*$
+^src/external_libs/compute/index.html$
+^src/external_libs/compute/.git$
+^src/external_libs/compute/.gitignore$
+^src/external_libs/compute/CONTRIBUTING.md$
+^src/external_libs/compute/README.md$
 src/external_libs/fast_double_parser/benchmarks
 src/external_libs/fast_double_parser/Makefile
 src/external_libs/fast_double_parser/.*\.md

build-cran-package.sh

@@ -45,18 +45,18 @@ mkdir -p ${EIGEN_R_DIR}
 
 modules="Cholesky Core Dense Eigenvalues Geometry Householder Jacobi LU QR SVD"
 for eigen_module in ${modules}; do
-    cp eigen/Eigen/${eigen_module} ${EIGEN_R_DIR}/${eigen_module}
+    cp external_libs/eigen/Eigen/${eigen_module} ${EIGEN_R_DIR}/${eigen_module}
     if [ ${eigen_module} != "Dense" ]; then
         mkdir -p ${EIGEN_R_DIR}/src/${eigen_module}/
-        cp -R eigen/Eigen/src/${eigen_module}/* ${EIGEN_R_DIR}/src/${eigen_module}/
+        cp -R external_libs/eigen/Eigen/src/${eigen_module}/* ${EIGEN_R_DIR}/src/${eigen_module}/
     fi
 done
 
 mkdir -p ${EIGEN_R_DIR}/src/misc
-cp -R eigen/Eigen/src/misc/* ${EIGEN_R_DIR}/src/misc/
+cp -R external_libs/eigen/Eigen/src/misc/* ${EIGEN_R_DIR}/src/misc/
 
 mkdir -p ${EIGEN_R_DIR}/src/plugins
-cp -R eigen/Eigen/src/plugins/* ${EIGEN_R_DIR}/src/plugins/
+cp -R external_libs/eigen/Eigen/src/plugins/* ${EIGEN_R_DIR}/src/plugins/
 
 cd ${TEMP_R_DIR}
 

build_r.R

@@ -206,18 +206,6 @@ result <- file.copy(
 )
 .handle_result(result)
 
-# compute/ is a submodule with boost, only needed if
-# building the R package with GPU support
-if (USING_GPU) {
-  result <- file.copy(
-    from = "compute/"
-    , to = sprintf("%s/", TEMP_SOURCE_DIR)
-    , recursive = TRUE
-    , overwrite = TRUE
-  )
-  .handle_result(result)
-}
-
 EIGEN_R_DIR <- file.path(TEMP_SOURCE_DIR, "include", "Eigen")
 dir.create(EIGEN_R_DIR)
 
@@ -235,7 +223,7 @@ eigen_modules <- c(
 )
 for (eigen_module in eigen_modules) {
   result <- file.copy(
-    from = file.path("eigen", "Eigen", eigen_module)
+    from = file.path("external_libs", "eigen", "Eigen", eigen_module)
     , to = EIGEN_R_DIR
     , recursive = FALSE
     , overwrite = TRUE
@@ -252,7 +240,7 @@ for (eigen_module in c(eigen_modules, "misc", "plugins")) {
   module_dir <- file.path(EIGEN_R_DIR, "src", eigen_module)
   dir.create(module_dir, recursive = TRUE)
   result <- file.copy(
-    from = sprintf("%s/", file.path("eigen", "Eigen", "src", eigen_module))
+    from = sprintf("%s/", file.path("external_libs", "eigen", "Eigen", "src", eigen_module))
     , to = sprintf("%s/", file.path(EIGEN_R_DIR, "src"))
     , recursive = TRUE
     , overwrite = TRUE
@@ -305,13 +293,27 @@ result <- file.remove(
 #------------#
 # submodules #
 #------------#
+EXTERNAL_LIBS_R_DIR <- file.path(TEMP_SOURCE_DIR, "external_libs")
+dir.create(EXTERNAL_LIBS_R_DIR)
+for (submodule in list.dirs(
+  path = "external_libs"
+  , full.names = FALSE
+  , recursive = FALSE
+)) {
+  # compute/ is a submodule with boost, only needed if
+  # building the R package with GPU support;
+  # eigen/ has a special treatment due to licensing aspects
+  if ((submodule == "compute" && !USING_GPU) || submodule == "eigen") {
+    next
+  }
   result <- file.copy(
-  from = "external_libs/"
-  , to = sprintf("%s/", TEMP_SOURCE_DIR)
+    from = sprintf("%s/", file.path("external_libs", submodule))
+    , to = sprintf("%s/", EXTERNAL_LIBS_R_DIR)
     , recursive = TRUE
     , overwrite = TRUE
   )
   .handle_result(result)
+}
 
 # copy files into the place CMake expects
 for (src_file in c("lightgbm_R.cpp", "lightgbm_R.h", "R_object_helper.h")) {

docs/conf.py

@@ -220,7 +220,6 @@ def generate_doxygen_xml(app):
         "SKIP_FUNCTION_MACROS=NO",
         "SORT_BRIEF_DOCS=YES",
         "WARN_AS_ERROR=YES",
-        "EXCLUDE_PATTERNS=*/eigen/*"
     ]
     doxygen_input = '\n'.join(doxygen_args)
     doxygen_input = bytes(doxygen_input, "utf-8")

python-package/MANIFEST.in

@@ -6,32 +6,32 @@ include compile/CMakeLists.txt
 include compile/CMakeIntegratedOpenCL.cmake
 recursive-include compile *.so
 recursive-include compile/Release *.dll
-include compile/compute/CMakeLists.txt
-recursive-include compile/compute/cmake *
-recursive-include compile/compute/include *
-recursive-include compile/compute/meta *
-include compile/eigen/CMakeLists.txt
-include compile/eigen/Eigen/Cholesky
-include compile/eigen/Eigen/Core
-include compile/eigen/Eigen/Dense
-include compile/eigen/Eigen/Eigenvalues
-include compile/eigen/Eigen/Geometry
-include compile/eigen/Eigen/Householder
-include compile/eigen/Eigen/Jacobi
-include compile/eigen/Eigen/LU
-include compile/eigen/Eigen/QR
-include compile/eigen/Eigen/SVD
-recursive-include compile/eigen/Eigen/src/Cholesky *
-recursive-include compile/eigen/Eigen/src/Core *
-recursive-include compile/eigen/Eigen/src/Eigenvalues *
-recursive-include compile/eigen/Eigen/src/Geometry *
-recursive-include compile/eigen/Eigen/src/Householder *
-recursive-include compile/eigen/Eigen/src/Jacobi *
-recursive-include compile/eigen/Eigen/src/LU *
-recursive-include compile/eigen/Eigen/src/misc *
-recursive-include compile/eigen/Eigen/src/plugins *
-recursive-include compile/eigen/Eigen/src/QR *
-recursive-include compile/eigen/Eigen/src/SVD *
+include compile/external_libs/compute/CMakeLists.txt
+recursive-include compile/external_libs/compute/cmake *
+recursive-include compile/external_libs/compute/include *
+recursive-include compile/external_libs/compute/meta *
+include compile/external_libs/eigen/CMakeLists.txt
+include compile/external_libs/eigen/Eigen/Cholesky
+include compile/external_libs/eigen/Eigen/Core
+include compile/external_libs/eigen/Eigen/Dense
+include compile/external_libs/eigen/Eigen/Eigenvalues
+include compile/external_libs/eigen/Eigen/Geometry
+include compile/external_libs/eigen/Eigen/Householder
+include compile/external_libs/eigen/Eigen/Jacobi
+include compile/external_libs/eigen/Eigen/LU
+include compile/external_libs/eigen/Eigen/QR
+include compile/external_libs/eigen/Eigen/SVD
+recursive-include compile/external_libs/eigen/Eigen/src/Cholesky *
+recursive-include compile/external_libs/eigen/Eigen/src/Core *
+recursive-include compile/external_libs/eigen/Eigen/src/Eigenvalues *
+recursive-include compile/external_libs/eigen/Eigen/src/Geometry *
+recursive-include compile/external_libs/eigen/Eigen/src/Householder *
+recursive-include compile/external_libs/eigen/Eigen/src/Jacobi *
+recursive-include compile/external_libs/eigen/Eigen/src/LU *
+recursive-include compile/external_libs/eigen/Eigen/src/misc *
+recursive-include compile/external_libs/eigen/Eigen/src/plugins *
+recursive-include compile/external_libs/eigen/Eigen/src/QR *
+recursive-include compile/external_libs/eigen/Eigen/src/SVD *
 include compile/external_libs/fast_double_parser/CMakeLists.txt
 include compile/external_libs/fast_double_parser/LICENSE
 include compile/external_libs/fast_double_parser/LICENSE.BSL
@@ -44,4 +44,4 @@ recursive-include compile/src *
 recursive-include compile/windows LightGBM.sln LightGBM.vcxproj
 recursive-include compile/windows/x64/DLL *.dll
 global-exclude *.py[co]
-exclude compile/compute/.git
+exclude compile/external_libs/compute/.git

python-package/setup.py

@@ -65,8 +65,10 @@ def copy_files(integrated_opencl=False, use_gpu=False):
     if not os.path.isfile(os.path.join(CURRENT_DIR, '_IS_SOURCE_PACKAGE.txt')):
         copy_files_helper('include')
         copy_files_helper('src')
-        copy_files_helper('eigen')
-        copy_files_helper('external_libs')
+        for submodule in os.listdir(os.path.join(CURRENT_DIR, os.path.pardir, 'external_libs')):
+            if submodule == 'compute' and not use_gpu:
+                continue
+            copy_files_helper(os.path.join('external_libs', submodule))
         if not os.path.exists(os.path.join(CURRENT_DIR, "compile", "windows")):
             os.makedirs(os.path.join(CURRENT_DIR, "compile", "windows"))
         copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "windows", "LightGBM.sln"),
@@ -85,8 +87,6 @@ def copy_files(integrated_opencl=False, use_gpu=False):
             copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "CMakeIntegratedOpenCL.cmake"),
                       os.path.join(CURRENT_DIR, "compile", "CMakeIntegratedOpenCL.cmake"),
                       verbose=0)
-        if use_gpu:
-            copy_files_helper('compute')
 
 
 def clear_path(path):

windows/LightGBM.vcxproj

@@ -117,7 +117,7 @@
       <Optimization>Disabled</Optimization>
       <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
-      <AdditionalIncludeDirectories>$(ProjectDir)\..\eigen;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>$(ProjectDir)\..\external_libs\eigen;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
     </ClCompile>
     <Link>
       <AdditionalLibraryDirectories>
@@ -140,7 +140,7 @@
       <Optimization>Disabled</Optimization>
       <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
-      <AdditionalIncludeDirectories>$(ProjectDir)\..\eigen;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>$(ProjectDir)\..\external_libs\eigen;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
     </ClCompile>
     <Link>
       <AdditionalDependencies>
@@ -160,7 +160,7 @@
       <Optimization>Disabled</Optimization>
       <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
-      <AdditionalIncludeDirectories>$(ProjectDir)\..\eigen;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>$(ProjectDir)\..\external_libs\eigen;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
     </ClCompile>
     <Link>
       <AdditionalDependencies>
@@ -183,7 +183,7 @@
       <OmitFramePointers>true</OmitFramePointers>
       <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
-      <AdditionalIncludeDirectories>$(ProjectDir)\..\eigen;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>$(ProjectDir)\..\external_libs\eigen;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
     </ClCompile>
     <Link>
       <AdditionalLibraryDirectories>
@@ -210,7 +210,7 @@
       <OmitFramePointers>true</OmitFramePointers>
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
-      <AdditionalIncludeDirectories>$(ProjectDir)\..\eigen;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>$(ProjectDir)\..\external_libs\eigen;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
     </ClCompile>
     <Link>
       <AdditionalDependencies />
@@ -231,7 +231,7 @@
       <OmitFramePointers>true</OmitFramePointers>
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
-      <AdditionalIncludeDirectories>$(ProjectDir)\..\eigen;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>$(ProjectDir)\..\external_libs\eigen;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
     </ClCompile>
     <Link>
       <AdditionalDependencies>