This commit is contained in:
James Lamb 2022-01-19 22:36:12 -06:00
Родитель 7042755460
Коммит ece191f01e
5 изменённых файлов: 331 добавлений и 803 удалений

Просмотреть файл

@ -1,42 +0,0 @@
version: 3.3.2.99.{build}
image: Visual Studio 2015
platform: x64
configuration: # a trick to construct a build matrix with multiple Python versions
- 3.7
# only build pull requests and
# commits to 'master'
branches:
only:
- master
environment:
matrix:
- COMPILER: MSVC
TASK: python
- COMPILER: MINGW
TASK: python
clone_depth: 5
install:
- git submodule update --init --recursive # get `external_libs` folder
- set PATH=%PATH:C:\Program Files\Git\usr\bin;=% # delete sh.exe from PATH (mingw32-make fix)
- set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH%
- set PYTHON_VERSION=%CONFIGURATION%
- set CONDA_ENV="test-env"
- ps: |
switch ($env:PYTHON_VERSION) {
"3.7" {$env:MINICONDA = "C:\Miniconda37-x64"}
default {$env:MINICONDA = "C:\Miniconda37-x64"}
}
$env:PATH = "$env:MINICONDA;$env:MINICONDA\Scripts;$env:PATH"
$env:BUILD_SOURCESDIRECTORY = "$env:APPVEYOR_BUILD_FOLDER"
$env:LGB_VER = (Get-Content $env:APPVEYOR_BUILD_FOLDER\VERSION.txt).trim()
build: false
test_script:
- conda init powershell
- powershell.exe -ExecutionPolicy Bypass -File %APPVEYOR_BUILD_FOLDER%\.ci\test_windows.ps1

103
.github/workflows/cuda.yml поставляемый
Просмотреть файл

@ -1,103 +0,0 @@
name: CUDA Version
on:
push:
branches:
- master
pull_request:
branches:
- master
env:
github_actions: 'true'
os_name: linux
task: cuda
conda_env: test-env
jobs:
test:
name: cuda ${{ matrix.cuda_version }} ${{ matrix.method }} (linux, ${{ matrix.compiler }}, Python ${{ matrix.python_version }})
runs-on: [self-hosted, linux]
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
include:
- method: source
compiler: gcc
python_version: 3.7
cuda_version: "11.5.0"
- method: pip
compiler: clang
python_version: 3.8
cuda_version: "10.0"
- method: wheel
compiler: gcc
python_version: 3.9
cuda_version: "9.0"
steps:
- name: Setup or update software on host machine
run: |
sudo apt-get update
sudo apt-get install --no-install-recommends -y \
apt-transport-https \
ca-certificates \
curl \
git \
gnupg-agent \
lsb-release \
software-properties-common
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" -y
curl -sL https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add -
curl -sL https://nvidia.github.io/nvidia-docker/$(. /etc/os-release;echo $ID$VERSION_ID)/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list
sudo apt-get update
sudo apt-get install --no-install-recommends -y \
containerd.io \
docker-ce \
docker-ce-cli \
nvidia-docker2
sudo chmod a+rw /var/run/docker.sock
sudo systemctl restart docker
- name: Remove old folder with repository
run: sudo rm -rf $GITHUB_WORKSPACE
- name: Checkout repository
uses: actions/checkout@v1
with:
fetch-depth: 5
submodules: true
- name: Setup and run tests
run: |
export ROOT_DOCKER_FOLDER=/LightGBM
cat > docker.env <<EOF
GITHUB_ACTIONS=${{ env.github_actions }}
OS_NAME=${{ env.os_name }}
COMPILER=${{ matrix.compiler }}
TASK=${{ env.task }}
METHOD=${{ matrix.method }}
CONDA_ENV=${{ env.conda_env }}
PYTHON_VERSION=${{ matrix.python_version }}
BUILD_DIRECTORY=$ROOT_DOCKER_FOLDER
LGB_VER=$(head -n 1 VERSION.txt)
EOF
cat > docker-script.sh <<EOF
export CONDA=\$HOME/miniconda
export PATH=\$CONDA/bin:\$PATH
nvidia-smi
$ROOT_DOCKER_FOLDER/.ci/setup.sh || exit -1
$ROOT_DOCKER_FOLDER/.ci/test.sh || exit -1
EOF
cuda_version="${{ matrix.cuda_version }}"
cuda_major=${cuda_version%%.*}
docker_img="nvcr.io/nvidia/cuda:${cuda_version}-devel"
if [[ ${cuda_major} -gt 10 ]]; then
docker_img="${docker_img}-ubuntu$(lsb_release -rs)"
fi
docker run --env-file docker.env -v "$GITHUB_WORKSPACE":"$ROOT_DOCKER_FOLDER" --rm --gpus all "$docker_img" /bin/bash $ROOT_DOCKER_FOLDER/docker-script.sh
all-successful:
# https://github.community/t/is-it-possible-to-require-all-github-actions-tasks-to-pass-without-enumerating-them/117957/4?u=graingert
runs-on: ubuntu-latest
needs: [test]
steps:
- name: Note that all tests succeeded
run: echo "🎉"

79
.github/workflows/python_package.yml поставляемый
Просмотреть файл

@ -1,79 +0,0 @@
name: Python-package
on:
push:
branches:
- master
pull_request:
branches:
- master
env:
CONDA_ENV: test-env
GITHUB_ACTIONS: 'true'
jobs:
test:
name: ${{ matrix.task }} ${{ matrix.method }} (${{ matrix.os }}, Python ${{ matrix.python_version }})
runs-on: ${{ matrix.os }}
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
include:
- os: macOS-latest
task: regular
python_version: 3.8
- os: macOS-latest
task: sdist
python_version: 3.9
- os: macOS-latest
task: bdist
python_version: 3.8
- os: macOS-latest
task: if-else
python_version: 3.8
- os: macOS-latest
task: mpi
method: source
python_version: 3.9
- os: macOS-latest
task: mpi
method: pip
python_version: 3.9
- os: macOS-latest
task: mpi
method: wheel
python_version: 3.7
steps:
- name: Checkout repository
uses: actions/checkout@v2.4.0
with:
fetch-depth: 5
submodules: true
- name: Setup and run tests
shell: bash
run: |
export TASK="${{ matrix.task }}"
export METHOD="${{ matrix.method }}"
export PYTHON_VERSION="${{ matrix.python_version }}"
if [[ "${{ matrix.os }}" == "macOS-latest" ]]; then
export COMPILER="gcc"
export OS_NAME="macos"
elif [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then
export COMPILER="clang"
export OS_NAME="linux"
fi
export BUILD_DIRECTORY="$GITHUB_WORKSPACE"
export LGB_VER=$(head -n 1 VERSION.txt)
export CONDA=${HOME}/miniconda
export PATH=${CONDA}/bin:${PATH}
$GITHUB_WORKSPACE/.ci/setup.sh || exit -1
$GITHUB_WORKSPACE/.ci/test.sh || exit -1
all-successful:
# https://github.community/t/is-it-possible-to-require-all-github-actions-tasks-to-pass-without-enumerating-them/117957/4?u=graingert
runs-on: ubuntu-latest
needs: [test]
steps:
- name: Note that all tests succeeded
run: echo "🎉"

248
.github/workflows/r_package.yml поставляемый
Просмотреть файл

@ -1,248 +0,0 @@
name: R-package
on:
push:
branches:
- master
pull_request:
branches:
- master
env:
# hack to get around this:
# https://stat.ethz.ch/pipermail/r-package-devel/2020q3/005930.html
_R_CHECK_SYSTEM_CLOCK_: 0
# ignore R CMD CHECK NOTE checking how long it has
# been since the last submission
_R_CHECK_CRAN_INCOMING_REMOTE_: 0
# CRAN ignores the "installed size is too large" NOTE,
# so our CI can too. Setting to a large value here just
# to catch extreme problems
_R_CHECK_PKG_SIZES_THRESHOLD_: 100
jobs:
test:
name: ${{ matrix.task }} (${{ matrix.os }}, ${{ matrix.compiler }}, R ${{ matrix.r_version }}, ${{ matrix.build_type }})
runs-on: ${{ matrix.os }}
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
include:
################
# CMake builds #
################
- os: ubuntu-latest
task: r-package
compiler: gcc
r_version: 3.6
build_type: cmake
- os: ubuntu-latest
task: r-package
compiler: gcc
r_version: 4.0
build_type: cmake
- os: ubuntu-latest
task: r-package
compiler: clang
r_version: 3.6
build_type: cmake
- os: ubuntu-latest
task: r-package
compiler: clang
r_version: 4.0
build_type: cmake
- os: macOS-latest
task: r-package
compiler: gcc
r_version: 3.6
build_type: cmake
- os: macOS-latest
task: r-package
compiler: gcc
r_version: 4.0
build_type: cmake
- os: macOS-latest
task: r-package
compiler: clang
r_version: 3.6
build_type: cmake
- os: macOS-latest
task: r-package
compiler: clang
r_version: 4.0
build_type: cmake
- os: windows-latest
task: r-package
compiler: MINGW
toolchain: MINGW
r_version: 3.6
build_type: cmake
- os: windows-latest
task: r-package
compiler: MINGW
toolchain: MSYS
r_version: 4.0
build_type: cmake
# Visual Studio 2017
- os: windows-2016
task: r-package
compiler: MSVC
toolchain: MSVC
r_version: 3.6
build_type: cmake
# Visual Studio 2019
- os: windows-2019
task: r-package
compiler: MSVC
toolchain: MSVC
r_version: 4.0
build_type: cmake
# Visual Studio 2022
- os: windows-2022
task: r-package
compiler: MSVC
toolchain: MSVC
r_version: 4.0
build_type: cmake
###############
# CRAN builds #
###############
- os: windows-latest
task: r-package
compiler: MINGW
toolchain: MINGW
r_version: 3.6
build_type: cran
- os: windows-latest
task: r-package
compiler: MINGW
toolchain: MSYS
r_version: 4.0
build_type: cran
- os: ubuntu-latest
task: r-package
compiler: gcc
r_version: 4.0
build_type: cran
- os: macOS-latest
task: r-package
compiler: clang
r_version: 4.0
build_type: cran
################
# Other checks #
################
- os: ubuntu-latest
task: r-rchk
compiler: gcc
r_version: 4.0
build_type: cran
steps:
- name: Prevent conversion of line endings on Windows
if: startsWith(matrix.os, 'windows')
shell: pwsh
run: git config --global core.autocrlf false
- name: Checkout repository
uses: actions/checkout@v2.4.0
with:
fetch-depth: 5
submodules: true
- name: Install pandoc
uses: r-lib/actions/setup-pandoc@v1
- name: Setup and run tests on Linux and macOS
if: matrix.os == 'macOS-latest' || matrix.os == 'ubuntu-latest'
shell: bash
run: |
export TASK="${{ matrix.task }}"
export COMPILER="${{ matrix.compiler }}"
export GITHUB_ACTIONS="true"
if [[ "${{ matrix.os }}" == "macOS-latest" ]]; then
export OS_NAME="macos"
elif [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then
export OS_NAME="linux"
fi
export BUILD_DIRECTORY="$GITHUB_WORKSPACE"
export R_VERSION="${{ matrix.r_version }}"
export R_BUILD_TYPE="${{ matrix.build_type }}"
$GITHUB_WORKSPACE/.ci/setup.sh
$GITHUB_WORKSPACE/.ci/test.sh
- name: Setup and run tests on Windows
if: startsWith(matrix.os, 'windows')
shell: pwsh -command ". {0}"
run: |
$env:BUILD_SOURCESDIRECTORY = $env:GITHUB_WORKSPACE
$env:TOOLCHAIN = "${{ matrix.toolchain }}"
$env:R_VERSION = "${{ matrix.r_version }}"
$env:R_BUILD_TYPE = "${{ matrix.build_type }}"
$env:COMPILER = "${{ matrix.compiler }}"
$env:GITHUB_ACTIONS = "true"
$env:TASK = "${{ matrix.task }}"
& "$env:GITHUB_WORKSPACE/.ci/test_windows.ps1"
test-r-sanitizers:
name: r-sanitizers (ubuntu-latest, R-devel, ${{ matrix.compiler }} ASAN/UBSAN)
timeout-minutes: 60
runs-on: ubuntu-latest
container: wch1/r-debug
strategy:
fail-fast: false
matrix:
include:
- r_customization: san
compiler: gcc
- r_customization: csan
compiler: clang
steps:
- name: Checkout repository
uses: actions/checkout@v2.4.0
with:
fetch-depth: 5
submodules: true
- name: Install packages
shell: bash
run: |
RDscript${{ matrix.r_customization }} -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'Matrix', 'rmarkdown', 'testthat'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())"
sh build-cran-package.sh --r-executable=RD${{ matrix.r_customization }}
RD${{ matrix.r_customization }} CMD INSTALL lightgbm_*.tar.gz || exit -1
- name: Run tests with sanitizers
shell: bash
run: |
cd R-package/tests
exit_code=0
RDscript${{ matrix.r_customization }} testthat.R >> tests.log 2>&1 || exit_code=-1
cat ./tests.log
exit ${exit_code}
test-r-debian-clang:
name: r-package (debian, R-devel, clang)
timeout-minutes: 60
runs-on: ubuntu-latest
container: rhub/debian-clang-devel
steps:
- name: Install Git before checkout
shell: bash
run: |
apt-get update --allow-releaseinfo-change
apt-get install --no-install-recommends -y git
- name: Checkout repository
uses: actions/checkout@v2.4.0
with:
fetch-depth: 5
submodules: true
- name: Install packages and run tests
shell: bash
run: |
export PATH=/opt/R-devel/bin/:${PATH}
Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'Matrix', 'rmarkdown', 'testthat'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())"
sh build-cran-package.sh
R CMD check --as-cran --run-donttest lightgbm_*.tar.gz || exit -1
if grep -q -E "NOTE|WARNING|ERROR" lightgbm.Rcheck/00check.log; then
echo "NOTEs, WARNINGs, or ERRORs have been found by R CMD check"
exit -1
fi
all-successful:
# https://github.community/t/is-it-possible-to-require-all-github-actions-tasks-to-pass-without-enumerating-them/117957/4?u=graingert
runs-on: ubuntu-latest
needs: [test, test-r-sanitizers, test-r-debian-clang]
steps:
- name: Note that all tests succeeded
run: echo "🎉"

Просмотреть файл

@ -33,26 +33,26 @@ jobs:
container: ubuntu1404
strategy:
matrix:
regular:
TASK: regular
sdist:
TASK: sdist
PYTHON_VERSION: 3.7
bdist:
TASK: bdist
inference:
TASK: if-else
mpi_source:
TASK: mpi
METHOD: source
PYTHON_VERSION: 3.8
# regular:
# TASK: regular
# sdist:
# TASK: sdist
# PYTHON_VERSION: 3.7
# bdist:
# TASK: bdist
# inference:
# TASK: if-else
# mpi_source:
# TASK: mpi
# METHOD: source
# PYTHON_VERSION: 3.8
gpu_source:
TASK: gpu
METHOD: source
# on Ubuntu 14.04, gpu_source build segfaults with Python 3.9.9
PYTHON_VERSION: 3.9.6
swig:
TASK: swig
# swig:
# TASK: swig
steps:
- script: |
echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
@ -74,320 +74,320 @@ jobs:
pathtoPublish: '$(Build.ArtifactStagingDirectory)'
artifactName: PackageAssets
artifactType: container
###########################################
- job: Linux_latest
###########################################
variables:
COMPILER: clang
DEBIAN_FRONTEND: 'noninteractive'
IN_UBUNTU_LATEST_CONTAINER: 'true'
OS_NAME: 'linux'
SETUP_CONDA: 'true'
pool: sh-ubuntu
container: ubuntu-latest
strategy:
matrix:
regular:
TASK: regular
sdist:
TASK: sdist
bdist:
TASK: bdist
PYTHON_VERSION: 3.8
inference:
TASK: if-else
mpi_source:
TASK: mpi
METHOD: source
mpi_pip:
TASK: mpi
METHOD: pip
PYTHON_VERSION: 3.8
mpi_wheel:
TASK: mpi
METHOD: wheel
PYTHON_VERSION: 3.7
gpu_source:
TASK: gpu
METHOD: source
gpu_pip:
TASK: gpu
METHOD: pip
gpu_wheel:
TASK: gpu
METHOD: wheel
PYTHON_VERSION: 3.7
cpp_tests:
TASK: cpp-tests
METHOD: with-sanitizers
steps:
- script: |
echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
CONDA=$HOME/miniconda
echo "##vso[task.setvariable variable=CONDA]$CONDA"
echo "##vso[task.prependpath]$CONDA/bin"
AMDAPPSDK_PATH=$BUILD_SOURCESDIRECTORY/AMDAPPSDK
echo "##vso[task.setvariable variable=AMDAPPSDK_PATH]$AMDAPPSDK_PATH"
LD_LIBRARY_PATH=$AMDAPPSDK_PATH/lib/x86_64:$LD_LIBRARY_PATH
echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$LD_LIBRARY_PATH"
echo "##vso[task.setvariable variable=OPENCL_VENDOR_PATH]$AMDAPPSDK_PATH/etc/OpenCL/vendors"
displayName: 'Set variables'
# https://github.com/microsoft/azure-pipelines-agent/issues/2043#issuecomment-687983301
- script: |
/tmp/docker exec -t -u 0 ci-container \
sh -c "apt-get update && apt-get -o Dpkg::Options::="--force-confold" -y install sudo"
displayName: 'Install sudo'
- bash: $(Build.SourcesDirectory)/.ci/setup.sh
displayName: Setup
- bash: $(Build.SourcesDirectory)/.ci/test.sh
displayName: Test
###########################################
- job: QEMU_multiarch
###########################################
variables:
COMPILER: gcc
OS_NAME: 'linux'
PRODUCES_ARTIFACTS: 'true'
pool:
vmImage: ubuntu-latest
timeoutInMinutes: 180
strategy:
matrix:
bdist:
TASK: bdist
ARCH: aarch64
steps:
- script: |
sudo apt-get update
sudo apt-get install --no-install-recommends -y \
binfmt-support \
qemu \
qemu-user \
qemu-user-static
displayName: 'Install QEMU'
- script: |
docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
displayName: 'Enable Docker multi-architecture support'
- script: |
export ROOT_DOCKER_FOLDER=/LightGBM
cat > docker.env <<EOF
AZURE=$AZURE
OS_NAME=$OS_NAME
COMPILER=$COMPILER
TASK=$TASK
METHOD=$METHOD
CONDA_ENV=$CONDA_ENV
PYTHON_VERSION=$PYTHON_VERSION
BUILD_DIRECTORY=$ROOT_DOCKER_FOLDER
LGB_VER=$(head -n 1 VERSION.txt)
PRODUCES_ARTIFACTS=$PRODUCES_ARTIFACTS
BUILD_ARTIFACTSTAGINGDIRECTORY=$BUILD_ARTIFACTSTAGINGDIRECTORY
EOF
cat > docker-script.sh <<EOF
export CONDA=\$HOME/miniconda
export PATH=\$CONDA/bin:\$PATH
$ROOT_DOCKER_FOLDER/.ci/setup.sh || exit -1
$ROOT_DOCKER_FOLDER/.ci/test.sh || exit -1
EOF
IMAGE_URI="quay.io/pypa/manylinux2014_${ARCH}"
docker pull "${IMAGE_URI}" || exit -1
PLATFORM=$(docker inspect --format='{{.Os}}/{{.Architecture}}' "${IMAGE_URI}") || exit -1
echo "detected image platform: ${PLATFORM}"
docker run \
--platform "${PLATFORM}" \
--rm \
--env-file docker.env \
-v "$(Build.SourcesDirectory)":"$ROOT_DOCKER_FOLDER" \
-v "$(Build.ArtifactStagingDirectory)":"$(Build.ArtifactStagingDirectory)" \
"${IMAGE_URI}" \
/bin/bash $ROOT_DOCKER_FOLDER/docker-script.sh
displayName: 'Setup and run tests'
- task: PublishBuildArtifacts@1
condition: and(succeeded(), in(variables['TASK'], 'bdist'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
inputs:
pathtoPublish: '$(Build.ArtifactStagingDirectory)'
artifactName: PackageAssets
artifactType: container
###########################################
- job: macOS
###########################################
variables:
COMPILER: clang
OS_NAME: 'macos'
PRODUCES_ARTIFACTS: 'true'
pool:
vmImage: 'macOS-10.15'
strategy:
matrix:
regular:
TASK: regular
PYTHON_VERSION: 3.7
sdist:
TASK: sdist
PYTHON_VERSION: 3.8
bdist:
TASK: bdist
swig:
TASK: swig
cpp_tests:
TASK: cpp-tests
METHOD: with-sanitizers
SANITIZERS: "address;undefined"
steps:
- script: |
echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
CONDA=$AGENT_HOMEDIRECTORY/miniconda
echo "##vso[task.setvariable variable=CONDA]$CONDA"
echo "##vso[task.prependpath]$CONDA/bin"
echo "##vso[task.setvariable variable=JAVA_HOME]$JAVA_HOME_8_X64"
displayName: 'Set variables'
- bash: $(Build.SourcesDirectory)/.ci/setup.sh
displayName: Setup
- bash: $(Build.SourcesDirectory)/.ci/test.sh
displayName: Test
- task: PublishBuildArtifacts@1
condition: and(succeeded(), in(variables['TASK'], 'regular', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
inputs:
pathtoPublish: '$(Build.ArtifactStagingDirectory)'
artifactName: PackageAssets
artifactType: container
###########################################
- job: Windows
###########################################
pool:
vmImage: 'vs2017-win2016'
strategy:
matrix:
regular:
TASK: regular
sdist:
TASK: sdist
PYTHON_VERSION: 3.8
bdist:
TASK: bdist
swig:
TASK: swig
cpp_tests:
TASK: cpp-tests
steps:
- powershell: |
Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
displayName: 'Set Variables'
- script: |
cmd /c "powershell -ExecutionPolicy Bypass -File %BUILD_SOURCESDIRECTORY%/.ci/install_opencl.ps1"
condition: eq(variables['TASK'], 'bdist')
displayName: 'Install OpenCL'
- script: |
cmd /c "conda init powershell"
cmd /c "powershell -ExecutionPolicy Bypass -File %BUILD_SOURCESDIRECTORY%/.ci/test_windows.ps1"
displayName: Test
- task: PublishBuildArtifacts@1
condition: and(succeeded(), in(variables['TASK'], 'regular', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
inputs:
pathtoPublish: '$(Build.ArtifactStagingDirectory)'
artifactName: PackageAssets
artifactType: container
###########################################
- job: R_artifact
###########################################
condition: not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))
pool:
vmImage: 'ubuntu-latest'
container: rbase
steps:
- script: |
LGB_VER=$(head -n 1 VERSION.txt | sed "s/rc/-/g")
R_LIB_PATH=~/Rlib
export R_LIBS=${R_LIB_PATH}
mkdir -p ${R_LIB_PATH}
RDscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'Matrix', 'rmarkdown'), lib = '${R_LIB_PATH}', dependencies = c('Depends', 'Imports', 'LinkingTo'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" || exit -1
sh build-cran-package.sh --r-executable=RD || exit -1
mv lightgbm_${LGB_VER}.tar.gz $(Build.ArtifactStagingDirectory)/lightgbm-${LGB_VER}-r-cran.tar.gz
displayName: 'Build CRAN R-package'
- task: PublishBuildArtifacts@1
condition: succeeded()
inputs:
pathtoPublish: $(Build.ArtifactStagingDirectory)
artifactName: R-package
artifactType: container
# ###########################################
# - job: Linux_latest
# ###########################################
# variables:
# COMPILER: clang
# DEBIAN_FRONTEND: 'noninteractive'
# IN_UBUNTU_LATEST_CONTAINER: 'true'
# OS_NAME: 'linux'
# SETUP_CONDA: 'true'
# pool: sh-ubuntu
# container: ubuntu-latest
# strategy:
# matrix:
# regular:
# TASK: regular
# sdist:
# TASK: sdist
# bdist:
# TASK: bdist
# PYTHON_VERSION: 3.8
# inference:
# TASK: if-else
# mpi_source:
# TASK: mpi
# METHOD: source
# mpi_pip:
# TASK: mpi
# METHOD: pip
# PYTHON_VERSION: 3.8
# mpi_wheel:
# TASK: mpi
# METHOD: wheel
# PYTHON_VERSION: 3.7
# gpu_source:
# TASK: gpu
# METHOD: source
# gpu_pip:
# TASK: gpu
# METHOD: pip
# gpu_wheel:
# TASK: gpu
# METHOD: wheel
# PYTHON_VERSION: 3.7
# cpp_tests:
# TASK: cpp-tests
# METHOD: with-sanitizers
# steps:
# - script: |
# echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
# echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
# CONDA=$HOME/miniconda
# echo "##vso[task.setvariable variable=CONDA]$CONDA"
# echo "##vso[task.prependpath]$CONDA/bin"
# AMDAPPSDK_PATH=$BUILD_SOURCESDIRECTORY/AMDAPPSDK
# echo "##vso[task.setvariable variable=AMDAPPSDK_PATH]$AMDAPPSDK_PATH"
# LD_LIBRARY_PATH=$AMDAPPSDK_PATH/lib/x86_64:$LD_LIBRARY_PATH
# echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$LD_LIBRARY_PATH"
# echo "##vso[task.setvariable variable=OPENCL_VENDOR_PATH]$AMDAPPSDK_PATH/etc/OpenCL/vendors"
# displayName: 'Set variables'
# # https://github.com/microsoft/azure-pipelines-agent/issues/2043#issuecomment-687983301
# - script: |
# /tmp/docker exec -t -u 0 ci-container \
# sh -c "apt-get update && apt-get -o Dpkg::Options::="--force-confold" -y install sudo"
# displayName: 'Install sudo'
# - bash: $(Build.SourcesDirectory)/.ci/setup.sh
# displayName: Setup
# - bash: $(Build.SourcesDirectory)/.ci/test.sh
# displayName: Test
# ###########################################
# - job: QEMU_multiarch
# ###########################################
# variables:
# COMPILER: gcc
# OS_NAME: 'linux'
# PRODUCES_ARTIFACTS: 'true'
# pool:
# vmImage: ubuntu-latest
# timeoutInMinutes: 180
# strategy:
# matrix:
# bdist:
# TASK: bdist
# ARCH: aarch64
# steps:
# - script: |
# sudo apt-get update
# sudo apt-get install --no-install-recommends -y \
# binfmt-support \
# qemu \
# qemu-user \
# qemu-user-static
# displayName: 'Install QEMU'
# - script: |
# docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
# displayName: 'Enable Docker multi-architecture support'
# - script: |
# export ROOT_DOCKER_FOLDER=/LightGBM
# cat > docker.env <<EOF
# AZURE=$AZURE
# OS_NAME=$OS_NAME
# COMPILER=$COMPILER
# TASK=$TASK
# METHOD=$METHOD
# CONDA_ENV=$CONDA_ENV
# PYTHON_VERSION=$PYTHON_VERSION
# BUILD_DIRECTORY=$ROOT_DOCKER_FOLDER
# LGB_VER=$(head -n 1 VERSION.txt)
# PRODUCES_ARTIFACTS=$PRODUCES_ARTIFACTS
# BUILD_ARTIFACTSTAGINGDIRECTORY=$BUILD_ARTIFACTSTAGINGDIRECTORY
# EOF
# cat > docker-script.sh <<EOF
# export CONDA=\$HOME/miniconda
# export PATH=\$CONDA/bin:\$PATH
# $ROOT_DOCKER_FOLDER/.ci/setup.sh || exit -1
# $ROOT_DOCKER_FOLDER/.ci/test.sh || exit -1
# EOF
# IMAGE_URI="quay.io/pypa/manylinux2014_${ARCH}"
# docker pull "${IMAGE_URI}" || exit -1
# PLATFORM=$(docker inspect --format='{{.Os}}/{{.Architecture}}' "${IMAGE_URI}") || exit -1
# echo "detected image platform: ${PLATFORM}"
# docker run \
# --platform "${PLATFORM}" \
# --rm \
# --env-file docker.env \
# -v "$(Build.SourcesDirectory)":"$ROOT_DOCKER_FOLDER" \
# -v "$(Build.ArtifactStagingDirectory)":"$(Build.ArtifactStagingDirectory)" \
# "${IMAGE_URI}" \
# /bin/bash $ROOT_DOCKER_FOLDER/docker-script.sh
# displayName: 'Setup and run tests'
# - task: PublishBuildArtifacts@1
# condition: and(succeeded(), in(variables['TASK'], 'bdist'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
# inputs:
# pathtoPublish: '$(Build.ArtifactStagingDirectory)'
# artifactName: PackageAssets
# artifactType: container
# ###########################################
# - job: macOS
# ###########################################
# variables:
# COMPILER: clang
# OS_NAME: 'macos'
# PRODUCES_ARTIFACTS: 'true'
# pool:
# vmImage: 'macOS-10.15'
# strategy:
# matrix:
# regular:
# TASK: regular
# PYTHON_VERSION: 3.7
# sdist:
# TASK: sdist
# PYTHON_VERSION: 3.8
# bdist:
# TASK: bdist
# swig:
# TASK: swig
# cpp_tests:
# TASK: cpp-tests
# METHOD: with-sanitizers
# SANITIZERS: "address;undefined"
# steps:
# - script: |
# echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
# echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
# CONDA=$AGENT_HOMEDIRECTORY/miniconda
# echo "##vso[task.setvariable variable=CONDA]$CONDA"
# echo "##vso[task.prependpath]$CONDA/bin"
# echo "##vso[task.setvariable variable=JAVA_HOME]$JAVA_HOME_8_X64"
# displayName: 'Set variables'
# - bash: $(Build.SourcesDirectory)/.ci/setup.sh
# displayName: Setup
# - bash: $(Build.SourcesDirectory)/.ci/test.sh
# displayName: Test
# - task: PublishBuildArtifacts@1
# condition: and(succeeded(), in(variables['TASK'], 'regular', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
# inputs:
# pathtoPublish: '$(Build.ArtifactStagingDirectory)'
# artifactName: PackageAssets
# artifactType: container
# ###########################################
# - job: Windows
# ###########################################
# pool:
# vmImage: 'vs2017-win2016'
# strategy:
# matrix:
# regular:
# TASK: regular
# sdist:
# TASK: sdist
# PYTHON_VERSION: 3.8
# bdist:
# TASK: bdist
# swig:
# TASK: swig
# cpp_tests:
# TASK: cpp-tests
# steps:
# - powershell: |
# Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
# displayName: 'Set Variables'
# - script: |
# cmd /c "powershell -ExecutionPolicy Bypass -File %BUILD_SOURCESDIRECTORY%/.ci/install_opencl.ps1"
# condition: eq(variables['TASK'], 'bdist')
# displayName: 'Install OpenCL'
# - script: |
# cmd /c "conda init powershell"
# cmd /c "powershell -ExecutionPolicy Bypass -File %BUILD_SOURCESDIRECTORY%/.ci/test_windows.ps1"
# displayName: Test
# - task: PublishBuildArtifacts@1
# condition: and(succeeded(), in(variables['TASK'], 'regular', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
# inputs:
# pathtoPublish: '$(Build.ArtifactStagingDirectory)'
# artifactName: PackageAssets
# artifactType: container
# ###########################################
# - job: R_artifact
# ###########################################
# condition: not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))
# pool:
# vmImage: 'ubuntu-latest'
# container: rbase
# steps:
# - script: |
# LGB_VER=$(head -n 1 VERSION.txt | sed "s/rc/-/g")
# R_LIB_PATH=~/Rlib
# export R_LIBS=${R_LIB_PATH}
# mkdir -p ${R_LIB_PATH}
# RDscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'Matrix', 'rmarkdown'), lib = '${R_LIB_PATH}', dependencies = c('Depends', 'Imports', 'LinkingTo'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" || exit -1
# sh build-cran-package.sh --r-executable=RD || exit -1
# mv lightgbm_${LGB_VER}.tar.gz $(Build.ArtifactStagingDirectory)/lightgbm-${LGB_VER}-r-cran.tar.gz
# displayName: 'Build CRAN R-package'
# - task: PublishBuildArtifacts@1
# condition: succeeded()
# inputs:
# pathtoPublish: $(Build.ArtifactStagingDirectory)
# artifactName: R-package
# artifactType: container
###########################################
- job: Package
###########################################
dependsOn:
- Linux
- Linux_latest
- QEMU_multiarch
- macOS
- Windows
- R_artifact
condition: and(succeeded(), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
pool:
vmImage: 'ubuntu-latest'
steps:
# Create archives with complete source code included (with git submodules)
- task: ArchiveFiles@2
displayName: Create zip archive
condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
inputs:
rootFolderOrFile: $(Build.SourcesDirectory)
includeRootFolder: false
archiveType: zip
archiveFile: '$(Build.ArtifactStagingDirectory)/archives/LightGBM-complete_source_code_zip.zip'
replaceExistingArchive: true
- task: ArchiveFiles@2
displayName: Create tar.gz archive
condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
inputs:
rootFolderOrFile: $(Build.SourcesDirectory)
includeRootFolder: false
archiveType: tar
tarCompression: gz
archiveFile: '$(Build.ArtifactStagingDirectory)/archives/LightGBM-complete_source_code_tar_gz.tar.gz'
replaceExistingArchive: true
# Download all agent packages from all previous phases
- task: DownloadBuildArtifacts@0
displayName: Download package assets
inputs:
artifactName: PackageAssets
downloadPath: $(Build.SourcesDirectory)/binaries
- task: DownloadBuildArtifacts@0
displayName: Download R-package
condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
inputs:
artifactName: R-package
downloadPath: $(Build.SourcesDirectory)/R
- script: |
python "$(Build.SourcesDirectory)/.nuget/create_nuget.py" "$(Build.SourcesDirectory)/binaries/PackageAssets"
displayName: 'Create NuGet configuration files'
- task: NuGetCommand@2
inputs:
command: pack
packagesToPack: '$(Build.SourcesDirectory)/.nuget/*.nuspec'
packDestination: '$(Build.ArtifactStagingDirectory)/nuget'
- task: PublishBuildArtifacts@1
inputs:
pathtoPublish: '$(Build.ArtifactStagingDirectory)/nuget'
artifactName: NuGet
artifactType: container
- task: GitHubRelease@0
displayName: 'Create GitHub Release'
condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
inputs:
gitHubConnection: guolinke
repositoryName: '$(Build.Repository.Name)'
action: 'create'
target: '$(Build.SourceVersion)'
tagSource: 'auto'
title: '$(Build.SourceBranchName)'
assets: |
$(Build.SourcesDirectory)/binaries/PackageAssets/*
$(Build.SourcesDirectory)/R/R-package/*
$(Build.ArtifactStagingDirectory)/nuget/*.nupkg
$(Build.ArtifactStagingDirectory)/archives/*
assetUploadMode: 'delete'
isDraft: true
isPreRelease: false
addChangeLog: false
# ###########################################
# - job: Package
# ###########################################
# dependsOn:
# - Linux
# - Linux_latest
# - QEMU_multiarch
# - macOS
# - Windows
# - R_artifact
# condition: and(succeeded(), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
# pool:
# vmImage: 'ubuntu-latest'
# steps:
# # Create archives with complete source code included (with git submodules)
# - task: ArchiveFiles@2
# displayName: Create zip archive
# condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
# inputs:
# rootFolderOrFile: $(Build.SourcesDirectory)
# includeRootFolder: false
# archiveType: zip
# archiveFile: '$(Build.ArtifactStagingDirectory)/archives/LightGBM-complete_source_code_zip.zip'
# replaceExistingArchive: true
# - task: ArchiveFiles@2
# displayName: Create tar.gz archive
# condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
# inputs:
# rootFolderOrFile: $(Build.SourcesDirectory)
# includeRootFolder: false
# archiveType: tar
# tarCompression: gz
# archiveFile: '$(Build.ArtifactStagingDirectory)/archives/LightGBM-complete_source_code_tar_gz.tar.gz'
# replaceExistingArchive: true
# # Download all agent packages from all previous phases
# - task: DownloadBuildArtifacts@0
# displayName: Download package assets
# inputs:
# artifactName: PackageAssets
# downloadPath: $(Build.SourcesDirectory)/binaries
# - task: DownloadBuildArtifacts@0
# displayName: Download R-package
# condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
# inputs:
# artifactName: R-package
# downloadPath: $(Build.SourcesDirectory)/R
# - script: |
# python "$(Build.SourcesDirectory)/.nuget/create_nuget.py" "$(Build.SourcesDirectory)/binaries/PackageAssets"
# displayName: 'Create NuGet configuration files'
# - task: NuGetCommand@2
# inputs:
# command: pack
# packagesToPack: '$(Build.SourcesDirectory)/.nuget/*.nuspec'
# packDestination: '$(Build.ArtifactStagingDirectory)/nuget'
# - task: PublishBuildArtifacts@1
# inputs:
# pathtoPublish: '$(Build.ArtifactStagingDirectory)/nuget'
# artifactName: NuGet
# artifactType: container
# - task: GitHubRelease@0
# displayName: 'Create GitHub Release'
# condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
# inputs:
# gitHubConnection: guolinke
# repositoryName: '$(Build.Repository.Name)'
# action: 'create'
# target: '$(Build.SourceVersion)'
# tagSource: 'auto'
# title: '$(Build.SourceBranchName)'
# assets: |
# $(Build.SourcesDirectory)/binaries/PackageAssets/*
# $(Build.SourcesDirectory)/R/R-package/*
# $(Build.ArtifactStagingDirectory)/nuget/*.nupkg
# $(Build.ArtifactStagingDirectory)/archives/*
# assetUploadMode: 'delete'
# isDraft: true
# isPreRelease: false
# addChangeLog: false