From 691b842832f511a72b4aaab715b3cb6b00de8f90 Mon Sep 17 00:00:00 2001 From: Nikita Titov Date: Tue, 9 Apr 2019 05:23:32 +0300 Subject: [PATCH] [ci] update CI stuff (#2079) * updated boost submodule * updated docker with new stable Clang and CMake * switch to dev docker * updated setup script * updated MinGW on Appveyor * updated Azure config to use docker for GPU task * do not upgrade gcc - takes too long * test: switch compilers * switch compilers back * get back to main docker --- .appveyor.yml | 2 +- .ci/dockers/ubuntu-14.04/Dockerfile | 9 +++--- .ci/setup.sh | 21 ++++++-------- .vsts-ci.yml | 43 ++++++----------------------- compute | 2 +- 5 files changed, 24 insertions(+), 53 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 67e71698d..bc14fc0a0 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -15,7 +15,7 @@ clone_depth: 50 install: - git submodule update --init --recursive # get `compute` folder - set PATH=%PATH:C:\Program Files\Git\usr\bin;=% # delete sh.exe from PATH (mingw32-make fix) - - set PATH=C:\mingw-w64\x86_64-7.3.0-posix-seh-rt_v5-rev0\mingw64\bin;%PATH% + - set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH% - set PYTHON_VERSION=%CONFIGURATION% - ps: >- switch ($env:PYTHON_VERSION) { diff --git a/.ci/dockers/ubuntu-14.04/Dockerfile b/.ci/dockers/ubuntu-14.04/Dockerfile index a4b3476e2..6373e40a4 100644 --- a/.ci/dockers/ubuntu-14.04/Dockerfile +++ b/.ci/dockers/ubuntu-14.04/Dockerfile @@ -24,18 +24,19 @@ RUN apt-get update \ build-essential \ && rm -rf /var/lib/apt/lists/* -# Install clang 6.0 +# Install clang 7.0 RUN add-apt-repository ppa:ubuntu-toolchain-r/test -y \ && cd /tmp \ && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - \ - && add-apt-repository "deb http://apt.llvm.org/trusty/ llvm-toolchain-trusty-6.0 main" -y \ + && add-apt-repository "deb http://apt.llvm.org/trusty/ llvm-toolchain-trusty-7 main" -y \ && apt-get update \ && apt-get install -y --no-install-recommends \ - clang-6.0 \ + clang-7 \ + libomp-7-dev \ && rm -rf /var/lib/apt/lists/* # Install CMake -RUN curl -sL https://cmake.org/files/v3.13/cmake-3.13.2-Linux-x86_64.sh -o cmake.sh \ +RUN curl -sL https://cmake.org/files/v3.14/cmake-3.14.1-Linux-x86_64.sh -o cmake.sh \ && chmod +x cmake.sh \ && ./cmake.sh --prefix=/usr/local --exclude-subdir \ && rm cmake.sh diff --git a/.ci/setup.sh b/.ci/setup.sh index 775f3ad99..4ac70b35e 100755 --- a/.ci/setup.sh +++ b/.ci/setup.sh @@ -3,11 +3,10 @@ if [[ $OS_NAME == "macos" ]]; then if [[ $COMPILER == "clang" ]]; then brew install libomp - brew upgrade cmake # CMake >=3.12 is needed to find OpenMP at macOS if [[ $AZURE == "true" ]]; then sudo xcode-select -s /Applications/Xcode_8.3.3.app/Contents/Developer fi - else + else # gcc if [[ $TRAVIS == "true" ]]; then # rm '/usr/local/include/c++' # previous variant to deal with conflict link # brew cask uninstall oclint # reserve variant to deal with conflict link @@ -23,22 +22,18 @@ if [[ $OS_NAME == "macos" ]]; then wget -q -O conda.sh https://repo.continuum.io/miniconda/Miniconda${PYTHON_VERSION:0:1}-latest-MacOSX-x86_64.sh else # Linux if [[ $AZURE == "true" ]] && [[ $COMPILER == "clang" ]]; then - sudo apt-get update - sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-6.0 100 - sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-6.0 100 - sudo apt-get install --no-install-recommends -y libomp-dev + sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-7 100 + sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-7 100 fi if [[ $TASK == "mpi" ]]; then sudo apt-get update sudo apt-get install --no-install-recommends -y libopenmpi-dev openmpi-bin fi if [[ $TASK == "gpu" ]]; then - if [[ $TRAVIS == "true" ]]; then - sudo add-apt-repository ppa:kzemek/boost -y - fi + sudo add-apt-repository ppa:mhier/libboost-latest -y sudo apt-get update - sudo apt-get install --no-install-recommends -y libboost1.58-dev libboost-system1.58-dev libboost-filesystem1.58-dev ocl-icd-opencl-dev - cd $HOME_DIRECTORY + sudo apt-get install --no-install-recommends -y libboost1.68-dev ocl-icd-opencl-dev + cd $BUILD_DIRECTORY # to avoid permission errors wget -q https://github.com/Microsoft/LightGBM/releases/download/v2.0.12/AMD-APP-SDKInstaller-v3.0.130.136-GA-linux64.tar.bz2 tar -xjf AMD-APP-SDK*.tar.bz2 mkdir -p $OPENCL_VENDOR_PATH @@ -47,12 +42,12 @@ else # Linux mv $AMDAPPSDK_PATH/lib/x86_64/sdk/* $AMDAPPSDK_PATH/lib/x86_64/ echo libamdocl64.so > $OPENCL_VENDOR_PATH/amdocl64.icd fi - if [[ $TRAVIS == "true" ]] || [[ $TASK == "gpu" ]]; then + if [[ $TRAVIS == "true" ]]; then wget -q -O conda.sh https://repo.continuum.io/miniconda/Miniconda${PYTHON_VERSION:0:1}-latest-Linux-x86_64.sh fi fi -if [[ $TRAVIS == "true" ]] || [[ $OS_NAME == "macos" ]] || [[ $TASK == "gpu" ]]; then +if [[ $TRAVIS == "true" ]] || [[ $OS_NAME == "macos" ]]; then sh conda.sh -b -p $CONDA fi conda config --set always_yes yes --set changeps1 no diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 5c2650cd1..14a82afa8 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -15,7 +15,7 @@ jobs: vmImage: 'ubuntu-16.04' container: ubuntu1404 strategy: - maxParallel: 5 + maxParallel: 6 matrix: regular: TASK: regular @@ -31,35 +31,6 @@ jobs: TASK: mpi METHOD: source PYTHON_VERSION: 2.7 - steps: - - script: | - echo "##vso[task.setvariable variable=HOME_DIRECTORY]$AGENT_HOMEDIRECTORY" - echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" - echo "##vso[task.setvariable variable=OS_NAME]linux" - echo "##vso[task.setvariable variable=AZURE]true" - echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)" - echo "##vso[task.prependpath]$CONDA/bin" - displayName: 'Set variables' - - bash: $(Build.SourcesDirectory)/.ci/setup.sh - displayName: Setup - - bash: $(Build.SourcesDirectory)/.ci/test.sh - displayName: Test - - task: PublishBuildArtifacts@1 - condition: and(succeeded(), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))) - inputs: - pathtoPublish: '$(Build.ArtifactStagingDirectory)' - artifactName: PackageAssets - artifactType: container -########################################### -- job: Linux_gpu -########################################### - variables: - COMPILER: gcc - pool: - vmImage: 'ubuntu-16.04' - strategy: - maxParallel: 1 - matrix: gpu_source: TASK: gpu METHOD: source @@ -71,19 +42,23 @@ jobs: echo "##vso[task.setvariable variable=OS_NAME]linux" echo "##vso[task.setvariable variable=AZURE]true" echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)" - AMDAPPSDK_PATH=$AGENT_HOMEDIRECTORY/AMDAPPSDK + echo "##vso[task.prependpath]$CONDA/bin" + AMDAPPSDK_PATH=$BUILD_SOURCESDIRECTORY/AMDAPPSDK echo "##vso[task.setvariable variable=AMDAPPSDK_PATH]$AMDAPPSDK_PATH" LD_LIBRARY_PATH=$AMDAPPSDK_PATH/lib/x86_64:$LD_LIBRARY_PATH echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$LD_LIBRARY_PATH" echo "##vso[task.setvariable variable=OPENCL_VENDOR_PATH]$AMDAPPSDK_PATH/etc/OpenCL/vendors" - CONDA=$AGENT_HOMEDIRECTORY/miniconda - echo "##vso[task.setvariable variable=CONDA]$CONDA" - echo "##vso[task.prependpath]$CONDA/bin" displayName: 'Set variables' - bash: $(Build.SourcesDirectory)/.ci/setup.sh displayName: Setup - bash: $(Build.SourcesDirectory)/.ci/test.sh displayName: Test + - task: PublishBuildArtifacts@1 + condition: and(succeeded(), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))) + inputs: + pathtoPublish: '$(Build.ArtifactStagingDirectory)' + artifactName: PackageAssets + artifactType: container ########################################### - job: MacOS ########################################### diff --git a/compute b/compute index 509ebe4a9..36c89134d 160000 --- a/compute +++ b/compute @@ -1 +1 @@ -Subproject commit 509ebe4a9282eec8a92c65ce3bbc1925f1fdbe07 +Subproject commit 36c89134d4013b2e5e45bc55656a18bd6141995a