[docs][ci] added docs about GPU support out of the box for Windows wheels and small refactoring for dual test (#3660)

* added docs about GPU support out of the box for Windows and small refactoring for dual test * test * Revert "test" This reverts commit 45188103c2. * fix docs * fix docs * hotfix config * Apply suggestions from code review Co-authored-by: TP Boudreau <tpboudreau@gmail.com> Co-authored-by: TP Boudreau <tpboudreau@gmail.com>
2020-12-22 14:20:01 +03:00 · 2020-12-22 14:20:01 +03:00 · 1fb2744370
--- a/.appveyor.yml
+++ b/.appveyor.yml
@ -26,14 +26,15 @@ install:
  - set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH%
  - set PYTHON_VERSION=%CONFIGURATION%
  - set CONDA_ENV="test-env"
-  - ps: >-
+  - ps: |
      switch ($env:PYTHON_VERSION) {
          "3.6" {$env:MINICONDA = "C:\Miniconda36-x64"}
          "3.7" {$env:MINICONDA = "C:\Miniconda37-x64"}
          default {$env:MINICONDA = "C:\Miniconda37-x64"}
      }
-      $env:PATH="$env:MINICONDA;$env:MINICONDA\Scripts;$env:PATH"
-  - ps: $env:LGB_VER = (Get-Content $env:APPVEYOR_BUILD_FOLDER\VERSION.txt).trim()
+      $env:PATH = "$env:MINICONDA;$env:MINICONDA\Scripts;$env:PATH"
+      $env:BUILD_SOURCESDIRECTORY = "$env:APPVEYOR_BUILD_FOLDER"
+      $env:LGB_VER = (Get-Content $env:APPVEYOR_BUILD_FOLDER\VERSION.txt).trim()

 build: false

--- a/.ci/install_opencl.ps1
+++ b/.ci/install_opencl.ps1
@ -1,4 +1,3 @@
-
 Write-Output "Installing OpenCL CPU platform"

 $cache = "$env:PIPELINE_WORKSPACE\opencl_windows-amd_cpu-v3_0_130_135"
@ -6,7 +5,6 @@ $installer = "AMD-APP-SDKInstaller-v3.0.130.135-GA-windows-F-x64.exe"

 if ($env:OPENCL_INSTALLER_FOUND -ne 'true') {
  # Pipeline cache miss; download OpenCL platform installer executable into workspace cache
-
  Write-Output "Downloading OpenCL platform installer"
  Invoke-WebRequest -OutFile "$installer" -Uri "https://github.com/microsoft/LightGBM/releases/download/v2.0.12/$installer"

@ -25,7 +23,6 @@ if ($env:OPENCL_INSTALLER_FOUND -ne 'true') {
 }

 # Install OpenCL platform from installer executable expected in workspace cache
-
 Write-Output "Running OpenCL installer"
 Invoke-Command -ScriptBlock {Start-Process "$cache\$installer" -ArgumentList '/S /V"/quiet /norestart /passive /log opencl.log"' -Wait}

@ -42,4 +39,3 @@ if ($property -eq $null) {
  Write-Output "Current OpenCL drivers:"
  Write-Output $property
 }
-
--- a/.ci/test_windows.ps1
+++ b/.ci/test_windows.ps1
@ -6,15 +6,9 @@ function Check-Output {
  }
 }

-# Import the Chocolatey profile module so that the RefreshEnv command
-# invoked below properly updates the current PowerShell session enviroment.
-$module = "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
-Import-Module "$module" ; Check-Output $?
-
-# unify environment variables for Azure devops and AppVeyor
+# unify environment variable for Azure devops and AppVeyor
 if (Test-Path env:APPVEYOR) {
  $env:APPVEYOR = "true"
-  $env:BUILD_SOURCESDIRECTORY = $env:APPVEYOR_BUILD_FOLDER
 }

 if ($env:TASK -eq "r-package") {
@ -28,9 +22,11 @@ conda activate
 conda config --set always_yes yes --set changeps1 no
 conda update -q -y conda
 conda create -q -y -n $env:CONDA_ENV python=$env:PYTHON_VERSION joblib matplotlib numpy pandas psutil pytest python-graphviz scikit-learn scipy ; Check-Output $?
+if ($env:TASK -ne "bdist") {
+  conda activate $env:CONDA_ENV
+}

 if ($env:TASK -eq "regular") {
-  conda activate $env:CONDA_ENV
  mkdir $env:BUILD_SOURCESDIRECTORY/build; cd $env:BUILD_SOURCESDIRECTORY/build
  cmake -A x64 .. ; cmake --build . --target ALL_BUILD --config Release ; Check-Output $?
  cd $env:BUILD_SOURCESDIRECTORY/python-package
@ -39,7 +35,6 @@ if ($env:TASK -eq "regular") {
  cp $env:BUILD_SOURCESDIRECTORY/Release/lightgbm.exe $env:BUILD_ARTIFACTSTAGINGDIRECTORY
 }
 elseif ($env:TASK -eq "sdist") {
-  conda activate $env:CONDA_ENV
  cd $env:BUILD_SOURCESDIRECTORY/python-package
  python setup.py sdist --formats gztar ; Check-Output $?
  cd dist; pip install @(Get-ChildItem *.gz) -v ; Check-Output $?
@ -54,7 +49,12 @@ elseif ($env:TASK -eq "sdist") {
  cp $env:BUILD_SOURCESDIRECTORY/build/lightgbmlib.jar $env:BUILD_ARTIFACTSTAGINGDIRECTORY/lightgbmlib_win.jar
 }
 elseif ($env:TASK -eq "bdist") {
+  # Import the Chocolatey profile module so that the RefreshEnv command
+  # invoked below properly updates the current PowerShell session enviroment.
+  $module = "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
+  Import-Module "$module" ; Check-Output $?
  RefreshEnv
+
  Write-Output "Current OpenCL drivers:"
  Get-ItemProperty -Path Registry::HKEY_LOCAL_MACHINE\SOFTWARE\Khronos\OpenCL\Vendors

@ -64,7 +64,6 @@ elseif ($env:TASK -eq "bdist") {
  cd dist; pip install --user @(Get-ChildItem *.whl) ; Check-Output $?
  cp @(Get-ChildItem *.whl) $env:BUILD_ARTIFACTSTAGINGDIRECTORY
 } elseif (($env:APPVEYOR -eq "true") -and ($env:TASK -eq "python")) {
-  conda activate $env:CONDA_ENV
  cd $env:BUILD_SOURCESDIRECTORY\python-package
  if ($env:COMPILER -eq "MINGW") {
    python setup.py install --mingw ; Check-Output $?
@ -76,13 +75,14 @@ elseif ($env:TASK -eq "bdist") {
 if (($env:TASK -eq "sdist") -or (($env:APPVEYOR -eq "true") -and ($env:TASK -eq "python"))) {
  # cannot test C API with "sdist" task
  $tests = $env:BUILD_SOURCESDIRECTORY + "/tests/python_package_test"
-} elseif ($env:TASK -eq "bdist") {
-  $tests = $env:BUILD_SOURCESDIRECTORY + "/tests"
-  # Make sure we can do both CPU and GPU; see tests/python_package_test/test_dual.py
-  $env:LIGHTGBM_TEST_DUAL_CPU_GPU = "1"
 } else {
  $tests = $env:BUILD_SOURCESDIRECTORY + "/tests"
 }
+if ($env:TASK -eq "bdist") {
+  # Make sure we can do both CPU and GPU; see tests/python_package_test/test_dual.py
+  $env:LIGHTGBM_TEST_DUAL_CPU_GPU = "1"
+}
+
 pytest $tests ; Check-Output $?

 if (($env:TASK -eq "regular") -or (($env:APPVEYOR -eq "true") -and ($env:TASK -eq "python"))) {
--- a/.vsts-ci.yml
+++ b/.vsts-ci.yml
@ -129,11 +129,11 @@ jobs:
      key: '"opencl.windows" | "amd.cpu" | "v3.0.130.135" | "1"'
      path: $(Pipeline.Workspace)/opencl_windows-amd_cpu-v3_0_130_135
      cacheHitVar: OPENCL_INSTALLER_FOUND
-    condition: eq(variables.TASK, 'bdist')
-    displayName: 'Cache OpenCL'
+    condition: eq(variables['TASK'], 'bdist')
+    displayName: 'Cache OpenCL Installer'
  - script: |
      cmd /c "powershell -ExecutionPolicy Bypass -File %BUILD_SOURCESDIRECTORY%/.ci/install_opencl.ps1"
-    condition: eq(variables.TASK, 'bdist')
+    condition: eq(variables['TASK'], 'bdist')
    displayName: 'Install OpenCL'
  - script: |
      cmd /c "conda init powershell"
--- a/docs/GPU-Targets.rst
+++ b/docs/GPU-Targets.rst
@ -24,7 +24,7 @@ You can find below a table of correspondence:

 Legend:

-\* AMD APP SDK is deprecated. On Windows, OpenCL is included in AMD graphics driver. On Linux, newer generation AMD cards are supported by the `ROCm`_ driver. You can download an archived copy of AMD APP SDK for Linux from `our GitHub repo`_.
+\* AMD APP SDK is deprecated. On Windows, OpenCL is included in AMD graphics driver. On Linux, newer generation AMD cards are supported by the `ROCm`_ driver. You can download an archived copy of AMD APP SDK from our GitHub repo (`for Linux`_ and `for Windows`_).


 --------------
@ -159,7 +159,9 @@ Known issues:

 .. _ROCm: https://rocmdocs.amd.com/en/latest/

-.. _our GitHub repo: https://github.com/microsoft/LightGBM/releases/download/v2.0.12/AMD-APP-SDKInstaller-v3.0.130.136-GA-linux64.tar.bz2
+.. _for Linux: https://github.com/microsoft/LightGBM/releases/download/v2.0.12/AMD-APP-SDKInstaller-v3.0.130.136-GA-linux64.tar.bz2
+
+.. _for Windows: https://github.com/microsoft/LightGBM/releases/download/v2.0.12/AMD-APP-SDKInstaller-v3.0.130.135-GA-windows-F-x64.exe

 .. _NVIDIA CUDA Toolkit: https://developer.nvidia.com/cuda-downloads

--- a/docs/GPU-Windows.rst
+++ b/docs/GPU-Windows.rst
@ -77,9 +77,9 @@ OpenCL SDK Installation
 Installing the appropriate OpenCL SDK requires you to download the correct vendor source SDK.
 You need to know what you are going to use LightGBM!

-  For running on Intel, get `Intel SDK for OpenCL`_ (NOT RECOMMENDED).
+-  For running on Intel, get `Intel SDK for OpenCL`_ (**NOT RECOMMENDED**).

-  For running on AMD, get `AMD APP SDK`_ (you may want to replace the ``OpenCL.dll`` from GPU driver package with the one from the SDK, if the one shipped with the driver lacks some functions).
+-  For running on AMD, get AMD APP SDK (downloads `for Linux`_ and `for Windows`_). You may want to replace the ``OpenCL.dll`` from the GPU driver package with the one from the SDK, if the one shipped with the driver lacks some functions.

 -  For running on NVIDIA, get `CUDA Toolkit`_.

@ -573,7 +573,9 @@ And open an issue in GitHub `here`_ with that log.

 .. _CUDA Toolkit: https://developer.nvidia.com/cuda-downloads

-.. _AMD APP SDK: https://github.com/fireice-uk/xmr-stak/issues/1511
+.. _for Linux: https://github.com/microsoft/LightGBM/releases/download/v2.0.12/AMD-APP-SDKInstaller-v3.0.130.136-GA-linux64.tar.bz2
+
+.. _for Windows: https://github.com/microsoft/LightGBM/releases/download/v2.0.12/AMD-APP-SDKInstaller-v3.0.130.135-GA-windows-F-x64.exe

 .. _Khronos official OpenCL headers: https://github.com/KhronosGroup/OpenCL-Headers

--- a/python-package/README.rst
+++ b/python-package/README.rst
@ -23,6 +23,8 @@ Install from `PyPI <https://pypi.org/project/lightgbm>`_ Using ``pip``

 You may need to install `wheel <https://pythonwheels.com>`_ via ``pip install wheel`` first.

+Compiled library that is included in the wheel file supports both **GPU** and **CPU** versions out of the box. This feature is experimental and available only for **Windows** currently. To use **GPU** version you only need to install OpenCL Runtime libraries. For NVIDIA and AMD GPU they are included in the ordinary drivers for your graphics card, so no action is required. If you would like your AMD or Intel CPU to act like a GPU (for testing and debugging) you can install `AMD APP SDK <https://github.com/microsoft/LightGBM/releases/download/v2.0.12/AMD-APP-SDKInstaller-v3.0.130.135-GA-windows-F-x64.exe>`_.
+
 For **Windows** users, `VC runtime <https://support.microsoft.com/en-us/help/2977003/the-latest-supported-visual-c-downloads>`_ is needed if **Visual Studio** (2015 or newer) is not installed.

 For **Linux** users, **glibc** >= 2.14 is required.
--- a/tests/python_package_test/test_dual.py
+++ b/tests/python_package_test/test_dual.py
@ -1,3 +1,4 @@
+# coding: utf-8
 """Tests for dual GPU+CPU support."""

 import os
@ -5,32 +6,29 @@ import pytest

 import lightgbm as lgb
 import numpy as np
-from lightgbm.basic import LightGBMError
+
+from sklearn.metrics import log_loss
+
+from .utils import load_breast_cancer


@pytest.mark.skipif(
    os.environ.get("LIGHTGBM_TEST_DUAL_CPU_GPU", None) is None,
    reason="Only run if appropriate env variable is set",
 )
-def test_cpu_works():
-    """If compiled appropriately, the same installation will support both GPU and CPU."""
-    data = np.random.rand(500, 10)
-    label = np.random.randint(2, size=500)
-    validation_data = train_data = lgb.Dataset(data, label=label)
+def test_cpu_and_gpu_work():
+    # If compiled appropriately, the same installation will support both GPU and CPU.
+    X, y = load_breast_cancer(return_X_y=True)
+    data = lgb.Dataset(X, y)

-    param = {"verbosity": 2, "num_leaves": 31, "objective": "binary", "device": "cpu"}
-    gbm = lgb.train(param, train_data, 10, valid_sets=[validation_data])
+    params_cpu = {"verbosity": -1, "num_leaves": 31, "objective": "binary", "device": "cpu"}
+    cpu_bst = lgb.train(params_cpu, data, num_boost_round=10)
+    cpu_score = log_loss(y, cpu_bst.predict(X))

+    params_gpu = params_cpu.copy()
+    params_gpu["device"] = "gpu"
+    gpu_bst = lgb.train(params_gpu, data, num_boost_round=10)
+    gpu_score = log_loss(y, gpu_bst.predict(X))

-@pytest.mark.skipif(
-    os.environ.get("LIGHTGBM_TEST_DUAL_CPU_GPU", None) is None,
-    reason="Only run if appropriate env variable is set",
-)
-def test_gpu_works():
-    """If compiled appropriately, the same installation will support both GPU and CPU."""
-    data = np.random.rand(500, 10)
-    label = np.random.randint(2, size=500)
-    validation_data = train_data = lgb.Dataset(data, label=label)
-
-    param = {"verbosity": 2, "num_leaves": 31, "objective": "binary", "device": "gpu"}
-    gbm = lgb.train(param, train_data, 10, valid_sets=[validation_data])
+    np.testing.assert_allclose(cpu_score, gpu_score, rtol=1e-4)
+    assert gpu_score < 0.25