microsoft
/
LightGBM
зеркало из https://github.com/microsoft/LightGBM.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность
LightGBM/.vsts-ci.yml

482 строки
16 KiB
YAML
Исходник Постоянная ссылка Ответственный История

trigger:
branches:
include:
- master
tags:
include:
- v*
pr:
- master
variables:
AZURE: 'true'
CMAKE_BUILD_PARALLEL_LEVEL: 4
PYTHON_VERSION: '3.12'
runCodesignValidationInjection: false
skipComponentGovernanceDetection: true
Codeql.Enabled: false
Codeql.SkipTaskAutoInjection: true
DOTNET_CLI_TELEMETRY_OPTOUT: true
DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
SKBUILD_STRICT_CONFIG: true
resources:
# The __work/ directory, where Azure DevOps writes the source files, needs to be read-write because
# LightGBM's CI jobs write files in the source directory.
#
# For all the containers included here, all other directories that Azure mounts in are mounted as read-only
# to minimize the risk of side effects from one run affecting future runs.
# ref: https://learn.microsoft.com/en-us/azure/devops/pipelines/yaml-schema/resources-containers-container
containers:
- container: linux-artifact-builder
image: lightgbm/vsts-agent:manylinux_2_28_x86_64
mountReadOnly:
work: false
externals: true
tools: true
tasks: true
- container: ubuntu-latest
image: 'ubuntu:22.04'
options: "--name ci-container -v /usr/bin/docker:/tmp/docker:ro"
mountReadOnly:
work: false
externals: true
tools: true
tasks: true
- container: rbase
image: wch1/r-debug
mountReadOnly:
work: false
externals: true
tools: true
tasks: true
jobs:
###########################################
- job: Maintenance
###########################################
pool: mariner-20240410-0
container: ubuntu-latest
# routine maintenance (like periodically deleting old files),
# to be run on 1 random CI runner in the self-hosted pool each runner
steps:
- script: |
print-diagnostics(){
echo "---- df -h -m ----"
df -h -m
echo "---- docker system df ----"
/tmp/docker system df
echo "---- docker images ----"
/tmp/docker images
}
# check disk usage
print-diagnostics
# remove old containers, container images, volumes
# ref: https://stackoverflow.com/a/32723127/3986677)
echo "---- running 'docker system prune' ----"
/tmp/docker system prune \
--all \
--force \
--filter until=720h
# check disk usage again
print-diagnostics
displayName: clean
###########################################
- job: Linux
###########################################
variables:
COMPILER: gcc
SETUP_CONDA: 'false'
OS_NAME: 'linux'
PRODUCES_ARTIFACTS: 'true'
pool: mariner-20240410-0
container: linux-artifact-builder
strategy:
matrix:
regular:
TASK: regular
PYTHON_VERSION: '3.10'
sdist:
TASK: sdist
PYTHON_VERSION: '3.8'
bdist:
TASK: bdist
PYTHON_VERSION: '3.9'
inference:
TASK: if-else
mpi_source:
TASK: mpi
METHOD: source
PYTHON_VERSION: '3.9'
gpu_source:
TASK: gpu
METHOD: source
swig:
TASK: swig
steps:
- script: |
echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
echo "##vso[task.prependpath]/usr/lib64/openmpi/bin"
echo "##vso[task.prependpath]$CONDA/bin"
displayName: 'Set variables'
- script: |
git clean -d -f -x
displayName: 'Clean source directory'
- script: |
echo '$(Build.SourceVersion)' > '$(Build.ArtifactStagingDirectory)/commit.txt'
displayName: 'Add commit hash to artifacts archive'
- task: Bash@3
displayName: Setup
inputs:
filePath: $(Build.SourcesDirectory)/.ci/setup.sh
targetType: filePath
- task: Bash@3
displayName: Test
inputs:
filePath: $(Build.SourcesDirectory)/.ci/test.sh
targetType: filePath
- task: PublishBuildArtifacts@1
condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
inputs:
pathtoPublish: '$(Build.ArtifactStagingDirectory)'
artifactName: PackageAssets
artifactType: container
###########################################
- job: Linux_latest
###########################################
variables:
COMPILER: clang-17
DEBIAN_FRONTEND: 'noninteractive'
IN_UBUNTU_BASE_CONTAINER: 'true'
OS_NAME: 'linux'
SETUP_CONDA: 'true'
pool: mariner-20240410-0
container: ubuntu-latest
strategy:
matrix:
regular:
TASK: regular
sdist:
TASK: sdist
bdist:
TASK: bdist
PYTHON_VERSION: '3.10'
inference:
TASK: if-else
mpi_source:
TASK: mpi
METHOD: source
mpi_pip:
TASK: mpi
METHOD: pip
PYTHON_VERSION: '3.11'
mpi_wheel:
TASK: mpi
METHOD: wheel
PYTHON_VERSION: '3.9'
gpu_source:
TASK: gpu
METHOD: source
PYTHON_VERSION: '3.11'
gpu_pip:
TASK: gpu
METHOD: pip
PYTHON_VERSION: '3.10'
gpu_wheel:
TASK: gpu
METHOD: wheel
PYTHON_VERSION: '3.9'
cpp_tests:
TASK: cpp-tests
METHOD: with-sanitizers
steps:
- script: |
echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
CONDA=$HOME/miniforge
echo "##vso[task.setvariable variable=CONDA]$CONDA"
echo "##vso[task.prependpath]$CONDA/bin"
displayName: 'Set variables'
# https://github.com/microsoft/azure-pipelines-agent/issues/2043#issuecomment-687983301
- script: |
/tmp/docker exec -t -u 0 ci-container \
sh -c "apt-get update && apt-get -o Dpkg::Options::="--force-confold" -y install sudo"
displayName: 'Install sudo'
- script: |
sudo apt-get update
sudo apt-get install -y --no-install-recommends git
git clean -d -f -x
displayName: 'Clean source directory'
- task: Bash@3
displayName: Setup
inputs:
filePath: $(Build.SourcesDirectory)/.ci/setup.sh
targetType: 'filePath'
- task: Bash@3
displayName: Test
inputs:
filePath: $(Build.SourcesDirectory)/.ci/test.sh
targetType: 'filePath'
###########################################
- job: QEMU_multiarch
###########################################
variables:
BUILD_DIRECTORY: /LightGBM
COMPILER: gcc
PRODUCES_ARTIFACTS: 'true'
pool:
vmImage: ubuntu-22.04
timeoutInMinutes: 180
strategy:
matrix:
bdist:
TASK: bdist
ARCH: aarch64
steps:
- script: |
sudo apt-get update
sudo apt-get install --no-install-recommends -y \
binfmt-support \
qemu \
qemu-user \
qemu-user-static
displayName: 'Install QEMU'
- script: |
docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
displayName: 'Enable Docker multi-architecture support'
- script: |
git clean -d -f -x
displayName: 'Clean source directory'
- script: |
cat > docker-script.sh <<EOF
export CONDA=\$HOME/miniforge
export PATH=\$CONDA/bin:/opt/rh/llvm-toolset-7.0/root/usr/bin:\$PATH
export LD_LIBRARY_PATH=/opt/rh/llvm-toolset-7.0/root/usr/lib64:\$LD_LIBRARY_PATH
\$BUILD_DIRECTORY/.ci/setup.sh || exit 1
\$BUILD_DIRECTORY/.ci/test.sh || exit 1
EOF
IMAGE_URI="lightgbm/vsts-agent:manylinux2014_aarch64"
docker pull "${IMAGE_URI}" || exit 1
PLATFORM=$(docker inspect --format='{{.Os}}/{{.Architecture}}' "${IMAGE_URI}") || exit 1
echo "detected image platform: ${PLATFORM}"
docker run \
--platform "${PLATFORM}" \
--rm \
--env AZURE=true \
--env BUILD_ARTIFACTSTAGINGDIRECTORY=$BUILD_ARTIFACTSTAGINGDIRECTORY \
--env BUILD_DIRECTORY=$BUILD_DIRECTORY \
--env COMPILER=$COMPILER \
--env METHOD=$METHOD \
--env OS_NAME=linux \
--env PRODUCES_ARTIFACTS=$PRODUCES_ARTIFACTS \
--env PYTHON_VERSION=$PYTHON_VERSION \
--env TASK=$TASK \
-v "$(Build.SourcesDirectory)":"$BUILD_DIRECTORY" \
-v "$(Build.ArtifactStagingDirectory)":"$(Build.ArtifactStagingDirectory)" \
"${IMAGE_URI}" \
/bin/bash $BUILD_DIRECTORY/docker-script.sh
displayName: 'Setup and run tests'
- task: PublishBuildArtifacts@1
condition: and(succeeded(), in(variables['TASK'], 'bdist'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
inputs:
pathtoPublish: '$(Build.ArtifactStagingDirectory)'
artifactName: PackageAssets
artifactType: container
###########################################
- job: macOS
###########################################
variables:
COMPILER: clang
OS_NAME: 'macos'
PRODUCES_ARTIFACTS: 'true'
pool:
vmImage: 'macOS-12'
strategy:
matrix:
regular:
TASK: regular
PYTHON_VERSION: '3.10'
sdist:
TASK: sdist
PYTHON_VERSION: '3.9'
bdist:
TASK: bdist
swig:
TASK: swig
cpp_tests:
TASK: cpp-tests
METHOD: with-sanitizers
SANITIZERS: "address;undefined"
steps:
- script: |
echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
CONDA=$AGENT_HOMEDIRECTORY/miniforge
echo "##vso[task.setvariable variable=CONDA]$CONDA"
echo "##vso[task.prependpath]$CONDA/bin"
echo "##vso[task.setvariable variable=JAVA_HOME]$JAVA_HOME_8_X64"
displayName: 'Set variables'
- script: |
git clean -d -f -x
displayName: 'Clean source directory'
- task: Bash@3
displayName: Setup
inputs:
filePath: $(Build.SourcesDirectory)/.ci/setup.sh
targetType: filePath
- task: Bash@3
displayName: Test
inputs:
filePath: $(Build.SourcesDirectory)/.ci/test.sh
targetType: filePath
- task: PublishBuildArtifacts@1
condition: and(succeeded(), in(variables['TASK'], 'regular', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
inputs:
pathtoPublish: '$(Build.ArtifactStagingDirectory)'
artifactName: PackageAssets
artifactType: container
###########################################
- job: Windows
###########################################
pool:
vmImage: 'windows-2019'
strategy:
matrix:
regular:
TASK: regular
PYTHON_VERSION: '3.10'
sdist:
TASK: sdist
PYTHON_VERSION: '3.9'
bdist:
TASK: bdist
swig:
TASK: swig
cpp_tests:
TASK: cpp-tests
steps:
- powershell: |
Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
displayName: 'Set Variables'
- script: |
git clean -d -f -x
displayName: 'Clean source directory'
- script: |
cmd /c "powershell -ExecutionPolicy Bypass -File %BUILD_SOURCESDIRECTORY%/.ci/install-opencl.ps1"
condition: eq(variables['TASK'], 'bdist')
displayName: 'Install OpenCL'
- script: |
cmd /c "conda config --remove channels defaults"
cmd /c "conda config --add channels nodefaults"
cmd /c "conda config --add channels conda-forge"
cmd /c "conda config --set channel_priority strict"
cmd /c "conda init powershell"
cmd /c "powershell -ExecutionPolicy Bypass -File %BUILD_SOURCESDIRECTORY%/.ci/test-windows.ps1"
displayName: Test
- task: PublishBuildArtifacts@1
condition: and(succeeded(), in(variables['TASK'], 'regular', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
inputs:
pathtoPublish: '$(Build.ArtifactStagingDirectory)'
artifactName: PackageAssets
artifactType: container
###########################################
- job: R_artifact
###########################################
condition: not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))
pool:
vmImage: 'ubuntu-22.04'
container: rbase
steps:
- script: |
git clean -d -f -x
displayName: 'Clean source directory'
- script: |
LGB_VER=$(head -n 1 VERSION.txt | sed "s/rc/-/g")
R_LIB_PATH=~/Rlib
export R_LIBS=${R_LIB_PATH}
mkdir -p ${R_LIB_PATH}
RDscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'RhpcBLASctl'), lib = '${R_LIB_PATH}', dependencies = c('Depends', 'Imports', 'LinkingTo'), repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())" || exit 1
sh build-cran-package.sh --r-executable=RD || exit 1
mv lightgbm_${LGB_VER}.tar.gz $(Build.ArtifactStagingDirectory)/lightgbm-${LGB_VER}-r-cran.tar.gz
displayName: 'Build CRAN R-package'
- task: PublishBuildArtifacts@1
condition: succeeded()
inputs:
pathtoPublish: $(Build.ArtifactStagingDirectory)
artifactName: R-package
artifactType: container
###########################################
- job: Package
###########################################
dependsOn:
- Linux
- Linux_latest
- QEMU_multiarch
- macOS
- Windows
- R_artifact
condition: and(succeeded(), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
pool:
vmImage: 'ubuntu-22.04'
steps:
# Create archives with complete source code included (with git submodules)
- task: ArchiveFiles@2
displayName: Create zip archive
condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
inputs:
rootFolderOrFile: $(Build.SourcesDirectory)
includeRootFolder: false
archiveType: zip
archiveFile: '$(Build.ArtifactStagingDirectory)/archives/LightGBM-complete_source_code_zip.zip'
replaceExistingArchive: true
- task: ArchiveFiles@2
displayName: Create tar.gz archive
condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
inputs:
rootFolderOrFile: $(Build.SourcesDirectory)
includeRootFolder: false
archiveType: tar
tarCompression: gz
archiveFile: '$(Build.ArtifactStagingDirectory)/archives/LightGBM-complete_source_code_tar_gz.tar.gz'
replaceExistingArchive: true
# Download all agent packages from all previous phases
- task: DownloadBuildArtifacts@0
displayName: Download package assets
inputs:
artifactName: PackageAssets
downloadPath: $(Build.SourcesDirectory)/binaries
- task: DownloadBuildArtifacts@0
displayName: Download R-package
condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
inputs:
artifactName: R-package
downloadPath: $(Build.SourcesDirectory)/R
- script: |
python "$(Build.SourcesDirectory)/.ci/create-nuget.py" "$(Build.SourcesDirectory)/binaries/PackageAssets"
displayName: 'Create NuGet configuration files'
- task: NuGetCommand@2
inputs:
command: pack
packagesToPack: '$(Build.SourcesDirectory)/.ci/nuget/*.nuspec'
packDestination: '$(Build.ArtifactStagingDirectory)/nuget'
- task: PublishBuildArtifacts@1
inputs:
pathtoPublish: '$(Build.ArtifactStagingDirectory)/nuget'
artifactName: NuGet
artifactType: container
- task: GitHubRelease@0
displayName: 'Create GitHub Release'
condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
inputs:
gitHubConnection: guolinke
repositoryName: '$(Build.Repository.Name)'
action: 'create'
target: '$(Build.SourceVersion)'
tagSource: 'auto'
title: '$(Build.SourceBranchName)'
assets: |
$(Build.SourcesDirectory)/binaries/PackageAssets/*
$(Build.SourcesDirectory)/R/R-package/*
$(Build.ArtifactStagingDirectory)/nuget/*.nupkg
$(Build.ArtifactStagingDirectory)/archives/*
assetUploadMode: 'delete'
isDraft: true
isPreRelease: false
addChangeLog: false
Ссылка в новой задаче Показать git blame Копировать постоянную ссылку