From cb4972eeefe83d1558fb3c855ea25097e570ef7f Mon Sep 17 00:00:00 2001 From: Oliver Borchert Date: Mon, 18 Mar 2024 03:26:26 +0100 Subject: [PATCH] [ci] Fix R 3.6 tests, dask tests, compatibility with dask>=2024.3.1 (#6357) --- .ci/test.sh | 6 +++--- .ci/test_r_package.sh | 2 ++ .ci/test_r_package_windows.ps1 | 6 +++++- .github/workflows/r_package.yml | 3 +-- python-package/lightgbm/compat.py | 12 +++++++++++- tests/python_package_test/test_dask.py | 12 ++++++++---- 6 files changed, 30 insertions(+), 11 deletions(-) diff --git a/.ci/test.sh b/.ci/test.sh index 79b2748e4..9d9c6e653 100755 --- a/.ci/test.sh +++ b/.ci/test.sh @@ -126,9 +126,9 @@ fi # older versions of Dask are incompatible with pandas>=2.0, but not all conda packages' metadata accurately reflects that # # ref: https://github.com/microsoft/LightGBM/issues/6030 -CONSTRAINED_DEPENDENCIES="'dask-core>=2023.5.0' 'distributed>=2023.5.0' 'pandas>=2.0'" +CONSTRAINED_DEPENDENCIES="'dask>=2023.5.0' 'distributed>=2023.5.0' 'pandas>=2.0'" if [[ $PYTHON_VERSION == "3.7" ]]; then - CONSTRAINED_DEPENDENCIES="'dask-core' 'distributed' 'pandas<2.0'" + CONSTRAINED_DEPENDENCIES="'dask' 'distributed' 'pandas<2.0'" fi # including python=version[build=*cpython] to ensure that conda doesn't fall back to pypy @@ -322,7 +322,7 @@ matplotlib.use\(\"Agg\"\)\ # importing the library should succeed even if all optional dependencies are not present conda uninstall -n $CONDA_ENV --force --yes \ cffi \ - dask-core \ + dask \ distributed \ joblib \ matplotlib \ diff --git a/.ci/test_r_package.sh b/.ci/test_r_package.sh index 4fda67a1f..d303d0ad2 100755 --- a/.ci/test_r_package.sh +++ b/.ci/test_r_package.sh @@ -81,6 +81,8 @@ fi # Installing R precompiled for Mac OS 10.11 or higher if [[ $OS_NAME == "macos" ]]; then + brew update-reset --auto-update + brew update --auto-update if [[ $R_BUILD_TYPE == "cran" ]]; then brew install automake || exit 1 fi diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index c4ae84f49..6af4dbb04 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -91,7 +91,11 @@ if ($env:R_MAJOR_VERSION -eq "3") { $env:R_LIB_PATH = "$env:BUILD_SOURCESDIRECTORY/RLibrary" -replace '[\\]', '/' $env:R_LIBS = "$env:R_LIB_PATH" $env:PATH = "$env:RTOOLS_BIN;" + "$env:RTOOLS_MINGW_BIN;" + "$env:R_LIB_PATH/R/bin/x64;"+ $env:PATH -$env:CRAN_MIRROR = "https://cran.rstudio.com" +if ([version]$env:R_VERSION -lt [version]"4.0") { + $env:CRAN_MIRROR = "https://cran-archive.r-project.org" +} else { + $env:CRAN_MIRROR = "https://cran.rstudio.com" +} $env:MIKTEX_EXCEPTION_PATH = "$env:TEMP\miktex" # don't fail builds for long-running examples unless they're very long. diff --git a/.github/workflows/r_package.yml b/.github/workflows/r_package.yml index db3e3c8a7..387b8c52e 100644 --- a/.github/workflows/r_package.yml +++ b/.github/workflows/r_package.yml @@ -275,7 +275,6 @@ jobs: clang-version: - 16 - 17 - - 18 runs-on: ubuntu-latest container: rhub/debian-clang-devel env: @@ -316,7 +315,7 @@ jobs: all-r-package-jobs-successful: if: always() runs-on: ubuntu-latest - needs: [test, test-r-sanitizers, test-r-debian-clang] + needs: [test, test-r-debian-clang] steps: - name: Note that all tests succeeded uses: re-actors/alls-green@v1.2.2 diff --git a/python-package/lightgbm/compat.py b/python-package/lightgbm/compat.py index 965dd3325..7656a0458 100644 --- a/python-package/lightgbm/compat.py +++ b/python-package/lightgbm/compat.py @@ -164,7 +164,17 @@ try: from dask.distributed import Client, Future, default_client, wait DASK_INSTALLED = True -except ImportError: +# catching 'ValueError' here because of this: +# https://github.com/microsoft/LightGBM/issues/6365#issuecomment-2002330003 +# +# That's potentially risky as dask does some significant import-time processing, +# like loading configuration from environment variables and files, and catching +# ValueError here might hide issues with that config-loading. +# +# But in exchange, it's less likely that 'import lightgbm' will fail for +# dask-related reasons, which is beneficial for any workloads that are using +# lightgbm but not its Dask functionality. +except (ImportError, ValueError): DASK_INSTALLED = False dask_array_from_delayed = None # type: ignore[assignment] diff --git a/tests/python_package_test/test_dask.py b/tests/python_package_test/test_dask.py index 9fe4da18f..37d6db254 100644 --- a/tests/python_package_test/test_dask.py +++ b/tests/python_package_test/test_dask.py @@ -213,13 +213,17 @@ def _create_data(objective, n_samples=1_000, output="array", chunk_size=500, **k def _r2_score(dy_true, dy_pred): - numerator = ((dy_true - dy_pred) ** 2).sum(axis=0, dtype=np.float64) - denominator = ((dy_true - dy_true.mean(axis=0)) ** 2).sum(axis=0, dtype=np.float64) - return (1 - numerator / denominator).compute() + y_true = dy_true.compute() + y_pred = dy_pred.compute() + numerator = ((y_true - y_pred) ** 2).sum(axis=0) + denominator = ((y_true - y_true.mean(axis=0)) ** 2).sum(axis=0) + return 1 - numerator / denominator def _accuracy_score(dy_true, dy_pred): - return da.average(dy_true == dy_pred).compute() + y_true = dy_true.compute() + y_pred = dy_pred.compute() + return (y_true == y_pred).mean() def _constant_metric(y_true, y_pred):