[ci] use conda-forge in Linux and macOS CI jobs (#4953)

* [ci] use conda-forge in CI jobs (fixes #4948) * comment out more jobs * try reverting graphviz patch, running more cuda jobs * get graphviz from PyPI and try removing some patches for r-lintr * start running appveyor again * use conda-forge if using conda * fix commands * conda install graphviz * try newer openmp * pin below openmp 11.x * focus on gpu task * trying to narrow down error * maybe gcc11 is the issue * start adding other tests back * pin openmp too * maybe need to pin to gcc less than 10.x * pin libgfortran and libstdcxx as well * pin to gcc 9.3.0 * move constraints up to initial environment * add all CI jobs back * try installing python-graphviz separately * try new lightgbm/vsts-agent image * fix typo * test if pinning gcc for linux gpu_source build is still necessary * ok yes, pinning gcc is necessary * test if Linux gpu_source works with Python 3.9.6 * no special exception for Linux gpu_source job * pin to Python 3.9.6 in Linux gpu_source * try explicitly asking for libstdcxx-ng for every linux build * swap compilers * switch compilers back * revert accidental whitespace change * comment out CI * try Linux gpu_source with different Python versions * Revert "try Linux gpu_source with different Python versions" This reverts commit f6f63cbb9b. * Revert "comment out CI" This reverts commit ece191f01e. * remove libxml2 install, change CONDA path * avoid installing conda in rchk job * empty commit 1 * empty commit 2 * empty commit 3 * empty commit 4 * add more verbose logging around installation of python-graphviz * empty commit 1 * get mamba info * get more conda info * add another mamba info call * allow for other macOS environments in GHA configuration * Revert "allow for other macOS environments in GHA configuration" This reverts commit a3c7a19926. * get more logs from mamba * get Build.ArtifactsStagingDirectory * get more logs and try to force re-installing everything * clean cache after every step * remove --update-all and make logs less verbose * remove more print statements and uncomment jobs * test if conda-clean issue fixes segfaults for gpu_source * pin python version for gpu_source * empty commit 1 * use miniforge instead * empty commit 1 * Apply suggestions from code review * bring workarounds back * remove duplicated graphviz system-wide installation (reverts #4095, #4097, #4238) * empty commit 1 * empty commit 2 * empty commit 3 * empty commit 4 * empty commit 5 * empty commit 6 * empty commit 7 * empty commit 8 * empty commit 9 * empty commit 10 * empty commit 10 * empty commit 10 * empty commit 10 * empty commit 11 * one more try * try to downgrade Python version for Linux GPU job * swap compilers * Revert "swap compilers" This reverts commit f04dc27b17. Co-authored-by: Nikita Titov <nekit94-12@hotmail.com> Co-authored-by: Nikita Titov <nekit94-08@mail.ru>
2022-02-10 18:56:35 -06:00 · 2022-02-10 18:56:35 -06:00 · 3500cb67b9
--- a/.ci/setup.sh
+++ b/.ci/setup.sh
@ -17,8 +17,10 @@ if [[ $OS_NAME == "macos" ]]; then
    if [[ $TASK == "swig" ]]; then
        brew install swig
    fi
-    brew install graphviz
-    curl -sL -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
+    curl \
+        -sL \
+        -o miniforge.sh \
+        https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-MacOSX-x86_64.sh
 else  # Linux
    if [[ $IN_UBUNTU_LATEST_CONTAINER == "true" ]]; then
        # fixes error "unable to initialize frontend: Dialog"
@ -42,9 +44,6 @@ else  # Linux
            libicu66 \
            libssl1.1 \
            libunwind8 \
-            libxau6 \
-            libxext6 \
-            libxrender1 \
            locales \
            netcat \
            unzip \
@ -81,16 +80,11 @@ else  # Linux
        mv $AMDAPPSDK_PATH/lib/x86_64/sdk/* $AMDAPPSDK_PATH/lib/x86_64/
        echo libamdocl64.so > $OPENCL_VENDOR_PATH/amdocl64.icd
    fi
-    ARCH=$(uname -m)
    if [[ $TASK == "cuda" ]]; then
        echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
        apt-get update
        apt-get install --no-install-recommends -y \
            curl \
-            graphviz \
-            libxau6 \
-            libxext6 \
-            libxrender1 \
            lsb-release \
            software-properties-common
        if [[ $COMPILER == "clang" ]]; then
@ -103,29 +97,19 @@ else  # Linux
        apt-get update
        apt-get install --no-install-recommends -y \
            cmake
-    else
-        if [[ $ARCH != "x86_64" ]]; then
-            yum update -y
-            yum install -y \
-                graphviz
-        else
-            sudo apt-get update
-            sudo apt-get install --no-install-recommends -y \
-                graphviz
-        fi
    fi
    if [[ $SETUP_CONDA != "false" ]]; then
-        if [[ $ARCH == "x86_64" ]]; then
-            curl -sL -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
-        else
-            curl -sL -o conda.sh https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-${ARCH}.sh
-        fi
+        ARCH=$(uname -m)
+        curl \
+            -sL \
+            -o miniforge.sh \
+            https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-${ARCH}.sh
    fi
 fi

-if [[ "${TASK}" != "r-package" ]]; then
+if [[ "${TASK}" != "r-package" ]] && [[ "${TASK}" != "r-rchk" ]]; then
    if [[ $SETUP_CONDA != "false" ]]; then
-        sh conda.sh -b -p $CONDA
+        sh miniforge.sh -b -p $CONDA
    fi
    conda config --set always_yes yes --set changeps1 no
    conda update -q -y conda
--- a/.ci/test.sh
+++ b/.ci/test.sh
@ -62,16 +62,13 @@ fi

 if [[ $TASK == "lint" ]]; then
    conda install -q -y -n $CONDA_ENV \
+        cmakelint \
+        cpplint \
+        isort \
+        mypy \
        pycodestyle \
        pydocstyle \
-        r-stringi  # stringi needs to be installed separate from r-lintr to avoid issues like 'unable to load shared object stringi.so'
-    # r-xfun below has to be upgraded because lintr requires > 0.19 for that package
-    conda install -q -y -n $CONDA_ENV \
-        -c conda-forge \
-            libxml2 \
-            "r-xfun>=0.19" \
        "r-lintr>=2.0"
-    pip install --user cmakelint cpplint isort mypy
    echo "Linting Python code"
    pycodestyle --ignore=E501,W503 --exclude=./.nuget,./external_libs . || exit -1
    pydocstyle --convention=numpy --add-ignore=D105 --match-dir="^(?!^external_libs|test|example).*" --match="(?!^test_|setup).*\.py" . || exit -1
@ -114,8 +111,22 @@ if [[ $TASK == "swig" ]]; then
    exit 0
 fi

-conda install -q -y -n $CONDA_ENV cloudpickle "dask=2021.9.1" "distributed=2021.9.1" joblib matplotlib numpy pandas psutil pytest scikit-learn scipy
-pip install graphviz  # python-graphviz from Anaconda is not allowed to be installed with Python 3.9
+conda install -q -y -n $CONDA_ENV \
+    cloudpickle \
+    dask \
+    distributed \
+    joblib \
+    matplotlib \
+    numpy \
+    pandas \
+    psutil \
+    pytest \
+    scikit-learn \
+    scipy || exit -1
+
+# python-graphviz has to be installed separately to prevent conda from downgrading to pypy
+conda install -q -y -n $CONDA_ENV \
+    python-graphviz || exit -1

 if [[ $OS_NAME == "macos" ]] && [[ $COMPILER == "clang" ]]; then
    # fix "OMP: Error #15: Initializing libiomp5.dylib, but found libomp.dylib already initialized." (OpenMP library conflict due to conda's MKL)
@ -229,7 +240,11 @@ import matplotlib\
 matplotlib.use\(\"Agg\"\)\
 ' plot_example.py  # prevent interactive window mode
    sed -i'.bak' 's/graph.render(view=True)/graph.render(view=False)/' plot_example.py
-    conda install -q -y -n $CONDA_ENV h5py ipywidgets notebook  # requirements for examples
+    # requirements for examples
+    conda install -q -y -n $CONDA_ENV \
+        h5py \
+        ipywidgets \
+        notebook
    for f in *.py **/*.py; do python $f || exit -1; done  # run all examples
    cd $BUILD_DIRECTORY/examples/python-guide/notebooks
    sed -i'.bak' 's/INTERACTIVE = False/assert False, \\"Interactive mode disabled\\"/' interactive_plot_example.ipynb
--- a/.github/workflows/cuda.yml
+++ b/.github/workflows/cuda.yml
@ -81,7 +81,7 @@ jobs:
            LGB_VER=$(head -n 1 VERSION.txt)
            EOF
            cat > docker-script.sh <<EOF
-            export CONDA=\$HOME/miniconda
+            export CONDA=\$HOME/miniforge
            export PATH=\$CONDA/bin:\$PATH
            nvidia-smi
            $ROOT_DOCKER_FOLDER/.ci/setup.sh || exit -1
--- a/.github/workflows/linkchecker.yml
+++ b/.github/workflows/linkchecker.yml
@ -27,7 +27,7 @@ jobs:
      - name: Setup and run tests
        run: |
          export BUILD_DIRECTORY="$GITHUB_WORKSPACE"
-          export CONDA=${HOME}/miniconda
+          export CONDA=${HOME}/miniforge
          export PATH=${CONDA}/bin:${HOME}/.local/bin:${PATH}
          $GITHUB_WORKSPACE/.ci/setup.sh || exit -1
          $GITHUB_WORKSPACE/.ci/test.sh || exit -1
--- a/.github/workflows/python_package.yml
+++ b/.github/workflows/python_package.yml
@ -66,7 +66,7 @@ jobs:
          fi
          export BUILD_DIRECTORY="$GITHUB_WORKSPACE"
          export LGB_VER=$(head -n 1 VERSION.txt)
-          export CONDA=${HOME}/miniconda
+          export CONDA=${HOME}/miniforge
          export PATH=${CONDA}/bin:${PATH}
          $GITHUB_WORKSPACE/.ci/setup.sh || exit -1
          $GITHUB_WORKSPACE/.ci/test.sh || exit -1
--- a/.github/workflows/static_analysis.yml
+++ b/.github/workflows/static_analysis.yml
@ -39,7 +39,7 @@ jobs:
        run: |
          export TASK="${{ matrix.task }}"
          export BUILD_DIRECTORY="$GITHUB_WORKSPACE"
-          export CONDA=${HOME}/miniconda
+          export CONDA=${HOME}/miniforge
          export PATH=${CONDA}/bin:$HOME/.local/bin:${PATH}
          $GITHUB_WORKSPACE/.ci/setup.sh || exit -1
          $GITHUB_WORKSPACE/.ci/test.sh || exit -1
--- a/.vsts-ci.yml
+++ b/.vsts-ci.yml
@ -14,7 +14,7 @@ variables:
 resources:
  containers:
  - container: ubuntu1404
-    image: lightgbm/vsts-agent:ubuntu-14.04
+    image: lightgbm/vsts-agent:ubuntu-14.04-dev
  - container: ubuntu-latest
    image: 'ubuntu:latest'
    options: "--name ci-container -v /usr/bin/docker:/tmp/docker:ro"
@ -49,6 +49,9 @@ jobs:
      gpu_source:
        TASK: gpu
        METHOD: source
+        # on Ubuntu 14.04, gpu_source build segfaults with newer version of Python
+        # (and newer version of scipy as a consequence)
+        PYTHON_VERSION: 3.7
      swig:
        TASK: swig
  steps:
@ -122,7 +125,7 @@ jobs:
  - script: |
      echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
      echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
-      CONDA=$HOME/miniconda
+      CONDA=$HOME/miniforge
      echo "##vso[task.setvariable variable=CONDA]$CONDA"
      echo "##vso[task.prependpath]$CONDA/bin"
      AMDAPPSDK_PATH=$BUILD_SOURCESDIRECTORY/AMDAPPSDK
@ -183,7 +186,7 @@ jobs:
      BUILD_ARTIFACTSTAGINGDIRECTORY=$BUILD_ARTIFACTSTAGINGDIRECTORY
      EOF
      cat > docker-script.sh <<EOF
-      export CONDA=\$HOME/miniconda
+      export CONDA=\$HOME/miniforge
      export PATH=\$CONDA/bin:\$PATH
      $ROOT_DOCKER_FOLDER/.ci/setup.sh || exit -1
      $ROOT_DOCKER_FOLDER/.ci/test.sh || exit -1
@ -236,7 +239,7 @@ jobs:
  - script: |
      echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
      echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
-      CONDA=$AGENT_HOMEDIRECTORY/miniconda
+      CONDA=$AGENT_HOMEDIRECTORY/miniforge
      echo "##vso[task.setvariable variable=CONDA]$CONDA"
      echo "##vso[task.prependpath]$CONDA/bin"
      echo "##vso[task.setvariable variable=JAVA_HOME]$JAVA_HOME_8_X64"