зеркало из https://github.com/microsoft/LightGBM.git
[ci][python] improved paths in setup.py and small CI refactoring (#1513)
* fixed paths in python-package installation * less cd commands at CI * hotfix * added copying missed file from windows directory * not copy filters file * refined paths in nuget creation script * removed filters file from MANIFEST.in
This commit is contained in:
Nikita Titov
GitHub
Родитель
6d7f1da349
Коммит
c6cdea75a3
|
|
@ -1,6 +1,6 @@
|
|||
version: 2.1.2.{build}
|
||||
version: 2.1.2.{build}
|
||||
|
||||
configuration: # A trick to construct a build matrix
|
||||
configuration: # a trick to construct a build matrix
|
||||
- 3.5
|
||||
- 3.6
|
||||
|
||||
|
|
@ -34,21 +34,18 @@ install:
|
|||
- activate test-env
|
||||
|
||||
build_script:
|
||||
- mkdir build && cd build
|
||||
- mkdir %APPVEYOR_BUILD_FOLDER%\build && cd %APPVEYOR_BUILD_FOLDER%\build
|
||||
- cmake -DCMAKE_GENERATOR_PLATFORM=x64 .. && cmake --build . --target ALL_BUILD --config Release
|
||||
- cd ..
|
||||
|
||||
test_script:
|
||||
- pytest tests/c_api_test/test_.py
|
||||
- cd python-package && python setup.py sdist --formats gztar
|
||||
- cd dist
|
||||
- pytest %APPVEYOR_BUILD_FOLDER%\tests\c_api_test\test_.py
|
||||
- cd %APPVEYOR_BUILD_FOLDER%\python-package && python setup.py sdist --formats gztar
|
||||
- IF "%COMPILER%"=="MINGW" (
|
||||
pip install lightgbm-%LGB_VER%.tar.gz --install-option=--mingw -v)
|
||||
pip install %APPVEYOR_BUILD_FOLDER%\python-package\dist\lightgbm-%LGB_VER%.tar.gz --install-option=--mingw -v)
|
||||
ELSE (
|
||||
pip install lightgbm-%LGB_VER%.tar.gz -v)
|
||||
- cd ../..
|
||||
- pytest tests/python_package_test
|
||||
- cd examples/python-guide
|
||||
pip install %APPVEYOR_BUILD_FOLDER%\python-package\dist\lightgbm-%LGB_VER%.tar.gz -v)
|
||||
- pytest %APPVEYOR_BUILD_FOLDER%\tests\python_package_test
|
||||
- cd %APPVEYOR_BUILD_FOLDER%\examples\python-guide
|
||||
- ps: >-
|
||||
@("import matplotlib", "matplotlib.use('Agg')") + (Get-Content "plot_example.py") | Set-Content "plot_example.py" # prevent interactive window mode
|
||||
- ps: >-
|
||||
|
|
@ -57,7 +54,7 @@ test_script:
|
|||
if ($LastExitCode -ne 0) { $host.SetShouldExit($LastExitCode) }
|
||||
} # run all examples
|
||||
- IF "%COMPILER%"=="MINGW" appveyor exit # skip all further steps
|
||||
- cd ../../python-package && python setup.py bdist_wheel --plat-name=win-amd64 --universal
|
||||
- cd %APPVEYOR_BUILD_FOLDER%\python-package && python setup.py bdist_wheel --plat-name=win-amd64 --universal
|
||||
|
||||
artifacts:
|
||||
- path: Release/lib_lightgbm.dll
|
||||
|
|
|
|||
|
|
@ -7,16 +7,19 @@ from distutils.file_util import copy_file
|
|||
if __name__ == "__main__":
|
||||
source = sys.argv[1]
|
||||
current_dir = os.path.abspath(os.path.dirname(__file__))
|
||||
if not os.path.exists(os.path.join(current_dir, "runtimes/linux-x64/native")):
|
||||
os.makedirs(os.path.join(current_dir, "runtimes/linux-x64/native"))
|
||||
if not os.path.exists(os.path.join(current_dir, "runtimes/osx-x64/native")):
|
||||
os.makedirs(os.path.join(current_dir, "runtimes/osx-x64/native"))
|
||||
if not os.path.exists(os.path.join(current_dir, "runtimes/win-x64/native")):
|
||||
os.makedirs(os.path.join(current_dir, "runtimes/win-x64/native"))
|
||||
copy_file(os.path.join(source, "lib_lightgbm.so"), os.path.join(current_dir, "runtimes/linux-x64/native/lib_lightgbm.so"))
|
||||
copy_file(os.path.join(source, "lib_lightgbm.dylib"), os.path.join(current_dir, "runtimes/osx-x64/native/lib_lightgbm.dylib"))
|
||||
copy_file(os.path.join(source, "lib_lightgbm.dll"), os.path.join(current_dir, "runtimes/win-x64/native/lib_lightgbm.dll"))
|
||||
version = open(os.path.join(current_dir, '../VERSION.txt')).read().strip()
|
||||
linux_folder_path = os.path.join(current_dir, "runtimes", "linux-x64", "native")
|
||||
if not os.path.exists(linux_folder_path):
|
||||
os.makedirs(linux_folder_path)
|
||||
osx_folder_path = os.path.join(current_dir, "runtimes", "osx-x64", "native")
|
||||
if not os.path.exists(osx_folder_path):
|
||||
os.makedirs(osx_folder_path)
|
||||
windows_folder_path = os.path.join(current_dir, "runtimes", "win-x64", "native")
|
||||
if not os.path.exists(windows_folder_path):
|
||||
os.makedirs(windows_folder_path)
|
||||
copy_file(os.path.join(source, "lib_lightgbm.so"), os.path.join(linux_folder_path, "lib_lightgbm.so"))
|
||||
copy_file(os.path.join(source, "lib_lightgbm.dylib"), os.path.join(osx_folder_path, "lib_lightgbm.dylib"))
|
||||
copy_file(os.path.join(source, "lib_lightgbm.dll"), os.path.join(windows_folder_path, "lib_lightgbm.dll"))
|
||||
version = open(os.path.join(current_dir, os.path.pardir, 'VERSION.txt')).read().strip()
|
||||
nuget_str = '''<?xml version="1.0"?>
|
||||
<package xmlns="http://schemas.microsoft.com/packaging/2013/05/nuspec.xsd">
|
||||
<metadata>
|
||||
|
|
@ -36,6 +39,6 @@ if __name__ == "__main__":
|
|||
<file src="runtimes\**" target="runtimes"/>
|
||||
</files>
|
||||
</package>
|
||||
''' % (version)
|
||||
''' % version
|
||||
with open(os.path.join(current_dir, "LightGBM.nuspec"), "w") as nuget_file:
|
||||
nuget_file.write(nuget_str)
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ fi
|
|||
|
||||
if [[ $TASK == "if-else" ]]; then
|
||||
conda install numpy
|
||||
mkdir build && cd build && cmake .. && make lightgbm || exit -1
|
||||
mkdir $TRAVIS_BUILD_DIR/build && cd $TRAVIS_BUILD_DIR/build && cmake .. && make lightgbm || exit -1
|
||||
cd $TRAVIS_BUILD_DIR/tests/cpp_test && ../../lightgbm config=train.conf convert_model_language=cpp convert_model=../../src/boosting/gbdt_prediction.cpp && ../../lightgbm config=predict.conf output_result=origin.pred || exit -1
|
||||
cd $TRAVIS_BUILD_DIR/build && make lightgbm || exit -1
|
||||
cd $TRAVIS_BUILD_DIR/tests/cpp_test && ../../lightgbm config=predict.conf output_result=ifelse.pred && python test.py || exit -1
|
||||
|
|
@ -85,7 +85,7 @@ if [[ $TASK == "gpu" ]]; then
|
|||
fi
|
||||
fi
|
||||
|
||||
mkdir build && cd build
|
||||
mkdir $TRAVIS_BUILD_DIR/build && cd $TRAVIS_BUILD_DIR/build
|
||||
|
||||
if [[ $TASK == "mpi" ]]; then
|
||||
cd $TRAVIS_BUILD_DIR/python-package && python setup.py sdist || exit -1
|
||||
|
|
|
|||
|
|
@ -7,6 +7,6 @@ recursive-include compile/Release *.dll
|
|||
recursive-include compile/compute *
|
||||
recursive-include compile/include *
|
||||
recursive-include compile/src *
|
||||
recursive-include compile/windows LightGBM.sln LightGBM.vcxproj LightGBM.vcxproj.filters
|
||||
recursive-include compile/windows LightGBM.sln LightGBM.vcxproj
|
||||
recursive-include compile/windows/x64/DLL *.dll
|
||||
global-exclude *.py[co]
|
||||
|
|
|
|||
|
|
@ -19,8 +19,7 @@ from setuptools.command.sdist import sdist
|
|||
|
||||
|
||||
def find_lib():
|
||||
CURRENT_DIR = os.path.dirname(__file__)
|
||||
libpath_py = os.path.join(CURRENT_DIR, 'lightgbm/libpath.py')
|
||||
libpath_py = os.path.join(CURRENT_DIR, 'lightgbm', 'libpath.py')
|
||||
libpath = {'__file__': libpath_py}
|
||||
exec(compile(open(libpath_py, "rb").read(), libpath_py, 'exec'), libpath, libpath)
|
||||
|
||||
|
|
@ -32,25 +31,29 @@ def find_lib():
|
|||
def copy_files(use_gpu=False):
|
||||
|
||||
def copy_files_helper(folder_name):
|
||||
src = os.path.join('..', folder_name)
|
||||
src = os.path.join(CURRENT_DIR, os.path.pardir, folder_name)
|
||||
if os.path.exists(src):
|
||||
dst = os.path.join('./compile', folder_name)
|
||||
dst = os.path.join(CURRENT_DIR, 'compile', folder_name)
|
||||
shutil.rmtree(dst, ignore_errors=True)
|
||||
distutils.dir_util.copy_tree(src, dst)
|
||||
else:
|
||||
raise Exception('Cannot copy {} folder'.format(src))
|
||||
raise Exception('Cannot copy {0} folder'.format(src))
|
||||
|
||||
if not os.path.isfile('./_IS_SOURCE_PACKAGE.txt'):
|
||||
if not os.path.isfile(os.path.join(CURRENT_DIR, '_IS_SOURCE_PACKAGE.txt')):
|
||||
copy_files_helper('include')
|
||||
copy_files_helper('src')
|
||||
if not os.path.exists("./compile/windows/"):
|
||||
os.makedirs("./compile/windows/")
|
||||
distutils.file_util.copy_file("../windows/LightGBM.sln", "./compile/windows/LightGBM.sln")
|
||||
distutils.file_util.copy_file("../windows/LightGBM.vcxproj", "./compile/windows/LightGBM.vcxproj")
|
||||
if not os.path.exists(os.path.join(CURRENT_DIR, "compile", "windows")):
|
||||
os.makedirs(os.path.join(CURRENT_DIR, "compile", "windows"))
|
||||
distutils.file_util.copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "windows", "LightGBM.sln"),
|
||||
os.path.join(CURRENT_DIR, "compile", "windows", "LightGBM.sln"))
|
||||
distutils.file_util.copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "windows", "LightGBM.vcxproj"),
|
||||
os.path.join(CURRENT_DIR, "compile", "windows", "LightGBM.vcxproj"))
|
||||
if use_gpu:
|
||||
copy_files_helper('compute')
|
||||
distutils.file_util.copy_file("../CMakeLists.txt", "./compile/")
|
||||
distutils.file_util.copy_file("../LICENSE", "./")
|
||||
distutils.file_util.copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "CMakeLists.txt"),
|
||||
os.path.join(CURRENT_DIR, "compile", "CMakeLists.txt"))
|
||||
distutils.file_util.copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "LICENSE"),
|
||||
os.path.join(CURRENT_DIR, "LICENSE"))
|
||||
|
||||
|
||||
def clear_path(path):
|
||||
|
|
@ -67,15 +70,15 @@ def clear_path(path):
|
|||
def silent_call(cmd, raise_error=False, error_msg=''):
|
||||
try:
|
||||
output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
|
||||
with open(path_log, "ab") as log:
|
||||
with open(LOG_PATH, "ab") as log:
|
||||
log.write(output)
|
||||
return 0
|
||||
except Exception as err:
|
||||
if isinstance(err, subprocess.CalledProcessError):
|
||||
with open(path_log, "ab") as log:
|
||||
with open(LOG_PATH, "ab") as log:
|
||||
log.write(err.output)
|
||||
if raise_error:
|
||||
raise Exception("\n".join((error_msg, log_notice)))
|
||||
raise Exception("\n".join((error_msg, LOG_NOTICE)))
|
||||
return 1
|
||||
|
||||
|
||||
|
|
@ -84,10 +87,10 @@ def compile_cpp(use_mingw=False, use_gpu=False, use_mpi=False, use_hdfs=False,
|
|||
boost_librarydir=None, opencl_include_dir=None,
|
||||
opencl_library=None):
|
||||
|
||||
if os.path.exists("build_cpp"):
|
||||
shutil.rmtree("build_cpp")
|
||||
os.makedirs("build_cpp")
|
||||
os.chdir("build_cpp")
|
||||
if os.path.exists(os.path.join(CURRENT_DIR, "build_cpp")):
|
||||
shutil.rmtree(os.path.join(CURRENT_DIR, "build_cpp"))
|
||||
os.makedirs(os.path.join(CURRENT_DIR, "build_cpp"))
|
||||
os.chdir(os.path.join(CURRENT_DIR, "build_cpp"))
|
||||
|
||||
logger.info("Starting to compile the library.")
|
||||
|
||||
|
|
@ -121,19 +124,20 @@ def compile_cpp(use_mingw=False, use_gpu=False, use_mpi=False, use_hdfs=False,
|
|||
error_msg='Please install MinGW first')
|
||||
else:
|
||||
status = 1
|
||||
lib_path = "../compile/windows/x64/DLL/lib_lightgbm.dll"
|
||||
lib_path = os.path.join(CURRENT_DIR, "compile", "windows", "x64", "DLL", "lib_lightgbm.dll")
|
||||
if not use_gpu and not use_hdfs:
|
||||
logger.info("Starting to compile with MSBuild from existing solution file.")
|
||||
platform_toolsets = ("v141", "v140")
|
||||
for pt in platform_toolsets:
|
||||
status = silent_call(["MSBuild", "../compile/windows/LightGBM.sln",
|
||||
status = silent_call(["MSBuild",
|
||||
os.path.join(CURRENT_DIR, "compile", "windows", "LightGBM.sln"),
|
||||
"/p:Configuration=DLL",
|
||||
"/p:Platform=x64",
|
||||
"/p:PlatformToolset={0}".format(pt)])
|
||||
if status == 0 and os.path.exists(lib_path):
|
||||
break
|
||||
else:
|
||||
clear_path("../compile/windows/x64")
|
||||
clear_path(os.path.join(CURRENT_DIR, "compile", "windows", "x64"))
|
||||
if status != 0 or not os.path.exists(lib_path):
|
||||
logger.warning("Compilation with MSBuild from existing solution file failed.")
|
||||
if status != 0 or not os.path.exists(lib_path):
|
||||
|
|
@ -144,10 +148,10 @@ def compile_cpp(use_mingw=False, use_gpu=False, use_mpi=False, use_hdfs=False,
|
|||
if status == 0:
|
||||
break
|
||||
else:
|
||||
clear_path("./")
|
||||
clear_path(os.path.join(CURRENT_DIR, "build_cpp"))
|
||||
if status != 0:
|
||||
raise Exception("\n".join(('Please install Visual Studio or MS Build and all required dependencies first',
|
||||
log_notice)))
|
||||
LOG_NOTICE)))
|
||||
silent_call(["cmake", "--build", ".", "--target", "_lightgbm", "--config", "Release"], raise_error=True,
|
||||
error_msg='Please install CMake first')
|
||||
else: # Linux, Darwin (macOS), etc.
|
||||
|
|
@ -155,7 +159,7 @@ def compile_cpp(use_mingw=False, use_gpu=False, use_mpi=False, use_hdfs=False,
|
|||
silent_call(cmake_cmd, raise_error=True, error_msg='Please install CMake and all required dependencies first')
|
||||
silent_call(["make", "_lightgbm"], raise_error=True,
|
||||
error_msg='An error has occurred while building lightgbm library file')
|
||||
os.chdir("..")
|
||||
os.chdir(CURRENT_DIR)
|
||||
|
||||
|
||||
class CustomInstallLib(install_lib):
|
||||
|
|
@ -200,7 +204,7 @@ class CustomInstall(install):
|
|||
self.precompile = 0
|
||||
|
||||
def run(self):
|
||||
open(path_log, 'wb').close()
|
||||
open(LOG_PATH, 'wb').close()
|
||||
if not self.precompile:
|
||||
copy_files(use_gpu=self.gpu)
|
||||
compile_cpp(use_mingw=self.mingw, use_gpu=self.gpu, use_mpi=self.mpi, use_hdfs=self.hdfs,
|
||||
|
|
@ -208,39 +212,40 @@ class CustomInstall(install):
|
|||
boost_include_dir=self.boost_include_dir, boost_librarydir=self.boost_librarydir,
|
||||
opencl_include_dir=self.opencl_include_dir, opencl_library=self.opencl_library)
|
||||
install.run(self)
|
||||
if os.path.isfile(path_log):
|
||||
os.remove(path_log)
|
||||
if os.path.isfile(LOG_PATH):
|
||||
os.remove(LOG_PATH)
|
||||
|
||||
|
||||
class CustomSdist(sdist):
|
||||
|
||||
def run(self):
|
||||
copy_files(use_gpu=True)
|
||||
open("./_IS_SOURCE_PACKAGE.txt", 'w').close()
|
||||
if os.path.exists("./lightgbm/Release/"):
|
||||
shutil.rmtree('./lightgbm/Release/')
|
||||
if os.path.exists("./lightgbm/windows/x64/"):
|
||||
shutil.rmtree('./lightgbm/windows/x64/')
|
||||
if os.path.isfile('./lightgbm/lib_lightgbm.so'):
|
||||
os.remove('./lightgbm/lib_lightgbm.so')
|
||||
open(os.path.join(CURRENT_DIR, '_IS_SOURCE_PACKAGE.txt'), 'w').close()
|
||||
if os.path.exists(os.path.join(CURRENT_DIR, 'lightgbm', 'Release')):
|
||||
shutil.rmtree(os.path.join(CURRENT_DIR, 'lightgbm', 'Release'))
|
||||
if os.path.exists(os.path.join(CURRENT_DIR, 'lightgbm', 'windows', 'x64')):
|
||||
shutil.rmtree(os.path.join(CURRENT_DIR, 'lightgbm', 'windows', 'x64'))
|
||||
if os.path.isfile(os.path.join(CURRENT_DIR, 'lightgbm', 'lib_lightgbm.so')):
|
||||
os.remove(os.path.join(CURRENT_DIR, 'lightgbm', 'lib_lightgbm.so'))
|
||||
sdist.run(self)
|
||||
if os.path.isfile('./_IS_SOURCE_PACKAGE.txt'):
|
||||
os.remove('./_IS_SOURCE_PACKAGE.txt')
|
||||
if os.path.isfile(os.path.join(CURRENT_DIR, '_IS_SOURCE_PACKAGE.txt')):
|
||||
os.remove(os.path.join(CURRENT_DIR, '_IS_SOURCE_PACKAGE.txt'))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if (8 * struct.calcsize("P")) != 64:
|
||||
raise Exception('Cannot install LightGBM in 32-bit Python, please use 64-bit Python instead.')
|
||||
|
||||
dir_path = os.path.dirname(os.path.realpath(__file__))
|
||||
path_log = os.path.join(os.path.expanduser('~'), 'LightGBM_compilation.log')
|
||||
log_notice = "The full version of error log was saved into {0}".format(path_log)
|
||||
if os.path.isfile(os.path.join('..', 'VERSION.txt')):
|
||||
distutils.file_util.copy_file(os.path.join('..', 'VERSION.txt'),
|
||||
os.path.join('.', 'lightgbm'))
|
||||
version = open(os.path.join(dir_path, 'lightgbm', 'VERSION.txt')).read().strip()
|
||||
CURRENT_DIR = os.path.abspath(os.path.dirname(__file__))
|
||||
LOG_PATH = os.path.join(os.path.expanduser('~'), 'LightGBM_compilation.log')
|
||||
LOG_NOTICE = "The full version of error log was saved into {0}".format(LOG_PATH)
|
||||
if os.path.isfile(os.path.join(CURRENT_DIR, os.path.pardir, 'VERSION.txt')):
|
||||
distutils.file_util.copy_file(os.path.join(CURRENT_DIR, os.path.pardir, 'VERSION.txt'),
|
||||
os.path.join(CURRENT_DIR, 'lightgbm', 'VERSION.txt'))
|
||||
version = open(os.path.join(CURRENT_DIR, 'lightgbm', 'VERSION.txt')).read().strip()
|
||||
readme = open(os.path.join(CURRENT_DIR, 'README.rst')).read()
|
||||
|
||||
sys.path.insert(0, '.')
|
||||
sys.path.insert(0, CURRENT_DIR)
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger('LightGBM')
|
||||
|
|
@ -248,7 +253,7 @@ if __name__ == "__main__":
|
|||
setup(name='lightgbm',
|
||||
version=version,
|
||||
description='LightGBM Python Package',
|
||||
long_description=open('README.rst').read(),
|
||||
long_description=readme,
|
||||
install_requires=[
|
||||
'numpy',
|
||||
'scipy',
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче