[ci] further simplify CI configurations (#6463)

2024-05-28 16:06:22 -05:00 · 2024-05-28 16:06:22 -05:00 · ceb9986192
--- a/.appveyor.yml
+++ b/.appveyor.yml
@ -2,15 +2,13 @@ version: 4.3.0.99.{build}

 image: Visual Studio 2015
 platform: x64
-configuration:  # a trick to construct a build matrix with multiple Python versions
+configuration:
  - '3.8'

-# only build pull requests and
-# commits to 'master' or any branch starting with 'release'
+# only build on 'master' and pull requests targeting it
 branches:
  only:
    - master
-    - /^release/

 environment:
  matrix:
@ -25,8 +23,9 @@ install:
  - git submodule update --init --recursive  # get `external_libs` folder
  - set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH%
  - set PYTHON_VERSION=%CONFIGURATION%
-  - set CONDA_ENV="test-env"
  - ps: |
+      $env:ALLOW_SKIP_ARROW_TESTS = "1"
+      $env:APPVEYOR = "true"
      $env:CMAKE_BUILD_PARALLEL_LEVEL = 4
      $env:MINICONDA = "C:\Miniconda3-x64"
      $env:PATH = "$env:MINICONDA;$env:MINICONDA\Scripts;$env:PATH"
--- a/.ci/check_python_dists.sh
+++ b/.ci/check_python_dists.sh
@ -26,11 +26,12 @@ fi
 PY_MINOR_VER=$(python -c "import sys; print(sys.version_info.minor)")
 if [ $PY_MINOR_VER -gt 7 ]; then
    echo "pydistcheck..."
-    pip install pydistcheck
+    pip install 'pydistcheck>=0.7.0'
    if { test "${TASK}" = "cuda" || test "${METHOD}" = "wheel"; }; then
        pydistcheck \
            --inspect \
-            --ignore 'compiled-objects-have-debug-symbols,distro-too-large-compressed' \
+            --ignore 'compiled-objects-have-debug-symbols'\
+            --ignore 'distro-too-large-compressed' \
            --max-allowed-size-uncompressed '100M' \
            --max-allowed-files 800 \
            ${DIST_DIR}/* || exit 1
--- a/.ci/get_workflow_status.py
+++ b/.ci/get_workflow_status.py
@ -10,11 +10,7 @@ import json
 from os import environ
 from sys import argv, exit
 from time import sleep
-
-try:
-    from urllib import request
-except ImportError:
-    import urllib2 as request
+from urllib import request


 def get_runs(trigger_phrase):
--- a/.ci/setup.sh
+++ b/.ci/setup.sh
@ -30,10 +30,6 @@ if [[ $OS_NAME == "macos" ]]; then
    if [[ $TASK == "swig" ]]; then
        brew install swig
    fi
-    curl \
-        -sL \
-        -o miniforge.sh \
-        https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-MacOSX-${ARCH}.sh
 else  # Linux
    if [[ $IN_UBUNTU_BASE_CONTAINER == "true" ]]; then
        # fixes error "unable to initialize frontend: Dialog"
@ -144,16 +140,14 @@ else  # Linux
        apt-get install --no-install-recommends -y \
            cmake
    fi
-    if [[ $SETUP_CONDA != "false" ]]; then
-        curl \
-            -sL \
-            -o miniforge.sh \
-            https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-${ARCH}.sh
-    fi
 fi

 if [[ "${TASK}" != "r-package" ]] && [[ "${TASK}" != "r-rchk" ]]; then
    if [[ $SETUP_CONDA != "false" ]]; then
+        curl \
+            -sL \
+            -o miniforge.sh \
+            https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-${ARCH}.sh
        sh miniforge.sh -b -p $CONDA
    fi
    conda config --set always_yes yes --set changeps1 no
--- a/.ci/test.sh
+++ b/.ci/test.sh
@ -3,6 +3,7 @@
 set -e -E -o -u pipefail

 # defaults
+CONDA_ENV="test-env"
 IN_UBUNTU_BASE_CONTAINER=${IN_UBUNTU_BASE_CONTAINER:-"false"}
 METHOD=${METHOD:-""}
 PRODUCES_ARTIFACTS=${PRODUCES_ARTIFACTS:-"false"}
--- a/.ci/test_r_package_valgrind.sh
+++ b/.ci/test_r_package_valgrind.sh
@ -72,10 +72,14 @@ bytes_possibly_lost=$(
    | tr -d ","
 )
 echo "valgrind found ${bytes_possibly_lost} bytes possibly lost"
-if [[ ${bytes_possibly_lost} -gt 1056 ]]; then
+if [[ ${bytes_possibly_lost} -gt 1104 ]]; then
    exit 1
 fi

+# ensure 'grep --count' doesn't cause failures
+set +e
+
+echo "checking for invalid reads"
 invalid_reads=$(
  cat ${VALGRIND_LOGS_FILE} \
    | grep --count -i "Invalid read"
@ -85,6 +89,7 @@ if [[ ${invalid_reads} -gt 0 ]]; then
    exit 1
 fi

+echo "checking for invalid writes"
 invalid_writes=$(
  cat ${VALGRIND_LOGS_FILE} \
    | grep --count -i "Invalid write"
--- a/.ci/test_windows.ps1
+++ b/.ci/test_windows.ps1
@ -6,14 +6,9 @@ function Check-Output {
  }
 }

+$env:CONDA_ENV = "test-env"
 $env:LGB_VER = (Get-Content $env:BUILD_SOURCESDIRECTORY\VERSION.txt).trim()

-# unify environment variable for Azure DevOps and AppVeyor
-if (Test-Path env:APPVEYOR) {
-  $env:APPVEYOR = "true"
-  $env:ALLOW_SKIP_ARROW_TESTS = "1"
-}
-
 if ($env:TASK -eq "r-package") {
  & $env:BUILD_SOURCESDIRECTORY\.ci\test_r_package_windows.ps1 ; Check-Output $?
  Exit 0
--- a/.github/workflows/cuda.yml
+++ b/.github/workflows/cuda.yml
@ -7,7 +7,6 @@ on:
  pull_request:
    branches:
    - master
-    - release/*
  # Run manually by clicking a button in the UI
  workflow_dispatch:
    inputs:
@ -59,16 +58,15 @@ jobs:
        run: |
          exit 0
  test:
-    name: ${{ matrix.task }} ${{ matrix.cuda_version }} ${{ matrix.method }} (linux, ${{ matrix.compiler }}, Python ${{ matrix.python_version }})
+    name: ${{ matrix.task }} ${{ matrix.cuda_version }} ${{ matrix.method }} (${{ matrix.linux_version }}, ${{ matrix.compiler }}, Python ${{ matrix.python_version }})
    runs-on: [self-hosted, linux]
    needs: [restart-docker]
    container:
-      image: ${{ matrix.image }}
+      image: nvcr.io/nvidia/cuda:${{ matrix.cuda_version }}-devel-${{ matrix.linux_version }}
      env:
        CMAKE_BUILD_PARALLEL_LEVEL: 4
        COMPILER: ${{ matrix.compiler }}
        CONDA: /tmp/miniforge
-        CONDA_ENV: test-env
        DEBIAN_FRONTEND: noninteractive
        METHOD: ${{ matrix.method }}
        OS_NAME: linux
@ -84,19 +82,19 @@ jobs:
            compiler: gcc
            python_version: "3.10"
            cuda_version: "11.8.0"
-            image: nvcr.io/nvidia/cuda:11.8.0-devel-ubuntu20.04
+            linux_version: "ubuntu20.04"
            task: cuda
          - method: source
            compiler: gcc
            python_version: "3.12"
            cuda_version: "12.2.0"
-            image: nvcr.io/nvidia/cuda:12.2.0-devel-ubuntu22.04
+            linux_version: "ubuntu22.04"
            task: cuda
          - method: pip
            compiler: clang
            python_version: "3.11"
            cuda_version: "11.8.0"
-            image: nvcr.io/nvidia/cuda:11.8.0-devel-ubuntu20.04
+            linux_version: "ubuntu20.04"
            task: cuda
    steps:
      - name: Install latest git
--- a/.github/workflows/linkchecker.yml
+++ b/.github/workflows/linkchecker.yml
@ -8,7 +8,6 @@ on:
    - cron: '0 8 * * *'

 env:
-  CONDA_ENV: test-env
  OS_NAME: 'linux'
  PYTHON_VERSION: '3.12'
  TASK: 'check-links'
--- a/.github/workflows/optional_checks.yml
+++ b/.github/workflows/optional_checks.yml
@ -4,7 +4,6 @@ on:
  pull_request:
    branches:
      - master
-      - release/*

 jobs:
  all-optional-checks-successful:
--- a/.github/workflows/python_package.yml
+++ b/.github/workflows/python_package.yml
@ -7,7 +7,6 @@ on:
  pull_request:
    branches:
    - master
-    - release/*

 # automatically cancel in-progress builds if another commit is pushed
 concurrency:
@ -16,7 +15,6 @@ concurrency:

 env:
  CMAKE_BUILD_PARALLEL_LEVEL: 4
-  CONDA_ENV: test-env

 jobs:
  test:
--- a/.github/workflows/r_package.yml
+++ b/.github/workflows/r_package.yml
@ -7,7 +7,6 @@ on:
  pull_request:
    branches:
    - master
-    - release/*

 # automatically cancel in-progress builds if another commit is pushed
 concurrency:
--- a/.github/workflows/static_analysis.yml
+++ b/.github/workflows/static_analysis.yml
@ -9,7 +9,6 @@ on:
  pull_request:
    branches:
    - master
-    - release/*

 # automatically cancel in-progress builds if another commit is pushed
 concurrency:
@ -18,7 +17,6 @@ concurrency:

 env:
  COMPILER: 'gcc'
-  CONDA_ENV: test-env
  OS_NAME: 'linux'
  PYTHON_VERSION: '3.12'

--- a/.vsts-ci.yml
+++ b/.vsts-ci.yml
@ -7,12 +7,10 @@ trigger:
    - v*
 pr:
 - master
- release/*
 variables:
  AZURE: 'true'
  CMAKE_BUILD_PARALLEL_LEVEL: 4
  PYTHON_VERSION: '3.12'
-  CONDA_ENV: test-env
  runCodesignValidationInjection: false
  skipComponentGovernanceDetection: true
  DOTNET_CLI_TELEMETRY_OPTOUT: true
@ -187,8 +185,8 @@ jobs:
 - job: QEMU_multiarch
 ###########################################
  variables:
+    BUILD_DIRECTORY: /LightGBM
    COMPILER: gcc
-    OS_NAME: 'linux'
    PRODUCES_ARTIFACTS: 'true'
  pool:
    vmImage: ubuntu-22.04
@ -214,25 +212,12 @@ jobs:
      git clean -d -f -x
    displayName: 'Clean source directory'
  - script: |
-      export ROOT_DOCKER_FOLDER=/LightGBM
-      cat > docker.env <<EOF
-      AZURE=$AZURE
-      OS_NAME=$OS_NAME
-      COMPILER=$COMPILER
-      TASK=$TASK
-      METHOD=$METHOD
-      CONDA_ENV=$CONDA_ENV
-      PYTHON_VERSION=$PYTHON_VERSION
-      BUILD_DIRECTORY=$ROOT_DOCKER_FOLDER
-      PRODUCES_ARTIFACTS=$PRODUCES_ARTIFACTS
-      BUILD_ARTIFACTSTAGINGDIRECTORY=$BUILD_ARTIFACTSTAGINGDIRECTORY
-      EOF
      cat > docker-script.sh <<EOF
      export CONDA=\$HOME/miniforge
      export PATH=\$CONDA/bin:/opt/rh/llvm-toolset-7.0/root/usr/bin:\$PATH
      export LD_LIBRARY_PATH=/opt/rh/llvm-toolset-7.0/root/usr/lib64:\$LD_LIBRARY_PATH
-      $ROOT_DOCKER_FOLDER/.ci/setup.sh || exit 1
-      $ROOT_DOCKER_FOLDER/.ci/test.sh || exit 1
+      \$BUILD_DIRECTORY/.ci/setup.sh || exit 1
+      \$BUILD_DIRECTORY/.ci/test.sh || exit 1
      EOF
      IMAGE_URI="lightgbm/vsts-agent:manylinux2014_aarch64"
      docker pull "${IMAGE_URI}" || exit 1
@ -241,11 +226,19 @@ jobs:
      docker run \
        --platform "${PLATFORM}" \
        --rm \
-        --env-file docker.env \
-        -v "$(Build.SourcesDirectory)":"$ROOT_DOCKER_FOLDER" \
+        --env AZURE=true \
+        --env BUILD_ARTIFACTSTAGINGDIRECTORY=$BUILD_ARTIFACTSTAGINGDIRECTORY \
+        --env BUILD_DIRECTORY=$BUILD_DIRECTORY \
+        --env COMPILER=$COMPILER \
+        --env METHOD=$METHOD \
+        --env OS_NAME=linux \
+        --env PRODUCES_ARTIFACTS=$PRODUCES_ARTIFACTS \
+        --env PYTHON_VERSION=$PYTHON_VERSION \
+        --env TASK=$TASK \
+        -v "$(Build.SourcesDirectory)":"$BUILD_DIRECTORY" \
        -v "$(Build.ArtifactStagingDirectory)":"$(Build.ArtifactStagingDirectory)" \
        "${IMAGE_URI}" \
-        /bin/bash $ROOT_DOCKER_FOLDER/docker-script.sh
+        /bin/bash $BUILD_DIRECTORY/docker-script.sh
    displayName: 'Setup and run tests'
  - task: PublishBuildArtifacts@1
    condition: and(succeeded(), in(variables['TASK'], 'bdist'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))