split_CI (#1141)

.github/workflows/test_macos.yml

@@ -1,94 +0,0 @@
-# There are some issues (in the downloading data phase) on MacOS when running with other tests. So we split it into an individual config.
-name: Test MacOS
-
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
-
-jobs:
-  build:
-    timeout-minutes: 120
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [macos-11, macos-latest]
-        # not supporting 3.6 due to annotations is not supported https://stackoverflow.com/a/52890129
-        python-version: [3.7, 3.8]
-
-    steps:
-    - uses: actions/checkout@v2
-
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-
-    - name: Lint with Black
-      run: |
-        cd ..
-        python -m pip install pip --upgrade
-        python -m pip install wheel --upgrade
-        python -m pip install black
-        python -m black qlib -l 120 --check --diff
-    # Test Qlib installed with pip
-
-    - name: Check Qlib with flake8
-      run: |
-        pip install --upgrade pip
-        pip install flake8
-        flake8 --ignore=E501,F541,E266,E402,W503,E731,E203 --per-file-ignores="__init__.py:F401,F403" qlib
-
-    - name: Install Qlib with pip
-      run: |
-          python -m pip install numpy==1.19.5
-          python -m pip install pyqlib --ignore-installed ruamel.yaml numpy
-    - name: Make html with sphnix
-      run: |
-        pip install -U sphinx
-        pip install sphinx_rtd_theme readthedocs_sphinx_ext
-        pip install --exists-action=w --no-cache-dir -r docs/requirements.txt 
-        cd docs 
-        sphinx-build -b html . build
-        cd ..
-    - name: Install Lightgbm for MacOS
-      run: |
-        /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Microsoft/qlib/main/.github/brew_install.sh)"
-        HOMEBREW_NO_AUTO_UPDATE=1 brew install lightgbm
-        # FIX MacOS error: Segmentation fault
-        # reference: https://github.com/microsoft/LightGBM/issues/4229
-        wget https://raw.githubusercontent.com/Homebrew/homebrew-core/fb8323f2b170bd4ae97e1bac9bf3e2983af3fdb0/Formula/libomp.rb
-        brew unlink libomp
-        brew install libomp.rb
-    - name: Test data downloads
-      run: |
-        python scripts/get_data.py qlib_data --name qlib_data_simple --target_dir ~/.qlib/qlib_data/cn_data_simple --interval 1d --region cn
-        python -c "import os; userpath=os.path.expanduser('~'); os.rename(userpath + '/.qlib/qlib_data/cn_data_simple', userpath + '/.qlib/qlib_data/cn_data')"
-        azcopy copy https://qlibpublic.blob.core.windows.net/data /tmp/qlibpublic --recursive
-        mv /tmp/qlibpublic/data tests/.data
-    - name: Test workflow by config (install from pip)
-      run: |
-        python qlib/workflow/cli.py examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml
-        python -m pip uninstall -y pyqlib
-    # Test Qlib installed from source
-    - name: Install Qlib from source
-      run: |
-        python -m pip install --upgrade cython
-        python -m pip install numpy jupyter jupyter_contrib_nbextensions
-        python -m pip install -U scipy scikit-learn # installing without this line will cause errors on GitHub Actions, while instsalling locally won't
-        python -m pip install gym tianshou torch
-        pip install -e .
-    - name: Install test dependencies
-      run: |
-        python -m pip install --upgrade pip
-        python -m pip install -U pyopenssl idna
-        python -m pip install black pytest
-    - name: Unit tests with Pytest
-      run: |
-        pip install -r scripts/data_collector/pit/requirements.txt
-        cd tests
-        python -m pytest . --durations=0
-    - name: Test workflow by config (install from source)
-      run: |
-          python qlib/workflow/cli.py examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml

.github/workflows/test_qlib_from_pip.yml

@@ -0,0 +1,46 @@
+name: Test qlib from pip
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+    timeout-minutes: 120
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [windows-latest, ubuntu-18.04, ubuntu-20.04, macos-11, macos-latest]
+        # not supporting 3.6 due to annotations is not supported https://stackoverflow.com/a/52890129
+        python-version: [3.7, 3.8]
+
+    steps:
+    - name: Test qlib from pip
+      uses: actions/checkout@v2
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Update pip to the latest version
+      run: |
+        python -m pip install --upgrade pip
+
+    - name: Qlib installation test
+      run: |
+        python -m pip install pyqlib
+        # Specify the numpy version because the numpy upgrade caused the CI test to fail, 
+        # and this line of code will be removed when the next version of qlib is released.
+        python -m pip install "numpy<1.23"
+
+    - name: Downloads dependencies data
+      run: |
+        python scripts/get_data.py qlib_data --name qlib_data_simple --target_dir ~/.qlib/qlib_data/cn_data --interval 1d --region cn
+
+    - name: Test workflow by config
+      run: |
+        qrun examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml

.github/workflows/test_qlib_from_source.yml

@@ -1,4 +1,4 @@
-name: Test
+name: Test qlib from source
 
 on:
   push:
@@ -9,41 +9,60 @@ on:
 jobs:
   build:
     timeout-minutes: 120
+
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [windows-latest, ubuntu-18.04, ubuntu-20.04]
+        os: [windows-latest, ubuntu-18.04, ubuntu-20.04, macos-11, macos-latest]
         # not supporting 3.6 due to annotations is not supported https://stackoverflow.com/a/52890129
         python-version: [3.7, 3.8]
 
     steps:
-    - uses: actions/checkout@v2
+    - name: Test qlib from source
+      uses: actions/checkout@v2
 
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python-version }}
 
+    - name: Update pip to the latest version
+      run: |
+        python -m pip install --upgrade pip
+
+    - name: Installing pytorch for macos
+      if: ${{ matrix.os == 'macos-11' || matrix.os == 'macos-latest' }}
+      run: |
+        python -m pip install torch torchvision torchaudio
+        brew install libomp  # lightgbm dependencies
+
+    - name: Installing pytorch for ubuntu
+      if: ${{ matrix.os == 'ubuntu-18.04' || matrix.os == 'ubuntu-20.04' }}
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu
+
+    - name: Installing pytorch for windows
+      if: ${{ matrix.os == 'windows-latest' }}
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install torch torchvision torchaudio
+
+    - name: Set up Python tools
+      run: |
+        python -m pip install --upgrade cython
+        python -m pip install -e .[dev]
+
     - name: Lint with Black
       run: |
-        pip install --upgrade pip
-        pip install black wheel
-        black qlib -l 120 --check --diff
-
-    - name: Install Qlib with pip
-      run: |
-        pip install numpy==1.19.5 ruamel.yaml
-        pip install pyqlib --ignore-installed
+        black . -l 120 --check --diff
 
     - name: Make html with sphinx
       run: |
-        pip install -U sphinx
-        pip install sphinx_rtd_theme readthedocs_sphinx_ext
-        pip install --exists-action=w --no-cache-dir -r docs/requirements.txt 
         cd docs 
         sphinx-build -b html . build
         cd ..
-        
+
     # Check Qlib with pylint
     # TODO: These problems we will solve in the future. Important among them are: W0221, W0223, W0237, E1102
       # C0103: invalid-name
@@ -67,11 +86,9 @@ jobs:
       # W1309: f-string-without-interpolation
       # E1102: not-callable
       # E1136: unsubscriptable-object
-    # References for parameters: https://github.com/PyCQA/pylint/issues/4577#issuecomment-1000245962 
+    # References for parameters: https://github.com/PyCQA/pylint/issues/4577#issuecomment-1000245962
     - name: Check Qlib with pylint
       run: |
-        pip install --upgrade pip
-        pip install pylint
         pylint --disable=C0104,C0114,C0115,C0116,C0301,C0302,C0411,C0413,C1802,R0401,R0801,R0902,R0903,R0911,R0912,R0913,R0914,R0915,R1720,W0105,W0123,W0201,W0511,W0613,W1113,W1514,E0401,E1121,C0103,C0209,R0402,R1705,R1710,R1725,R1735,W0102,W0212,W0221,W0223,W0231,W0237,W0612,W0621,W0622,W0703,W1309,E1102,E1136 --const-rgx='[a-z_][a-z0-9_]{2,30}$' qlib --init-hook "import astroid; astroid.context.InferenceContext.max_inferred = 500"
 
     # The following flake8 error codes were ignored:
@@ -95,47 +112,29 @@ jobs:
         # Description: If there is whitespace before ":", it cannot pass the black check.
     - name: Check Qlib with flake8
       run: |
-        pip install --upgrade pip
-        pip install flake8
         flake8 --ignore=E501,F541,E266,E402,W503,E731,E203 --per-file-ignores="__init__.py:F401,F403" qlib
 
     # https://github.com/python/mypy/issues/10600
     - name: Check Qlib with mypy
       run: |
-        pip install mypy
         mypy qlib --install-types --non-interactive || true
-        mypy qlib
+        mypy qlib --verbose
 
     - name: Test data downloads
       run: |
-        python scripts/get_data.py qlib_data --name qlib_data_simple --target_dir ~/.qlib/qlib_data/cn_data_simple --interval 1d --region cn
-        python -c "import os; userpath=os.path.expanduser('~'); os.rename(userpath + '/.qlib/qlib_data/cn_data_simple', userpath + '/.qlib/qlib_data/cn_data')"
+        python scripts/get_data.py qlib_data --name qlib_data_simple --target_dir ~/.qlib/qlib_data/cn_data --interval 1d --region cn
         azcopy copy https://qlibpublic.blob.core.windows.net/data/rl /tmp/qlibpublic/data --recursive
         mv /tmp/qlibpublic/data tests/.data
 
-    - name: Test workflow by config (install from pip)
+    - name: Test workflow by config (install from source)
       run: |
+        # Version 0.52.0 of numba must be installed manually in CI, otherwise it will cause incompatibility with the latest version of numpy.
+        python -m pip install numba==0.52.0
+        # You must update numpy manually, because when installing python tools, it will try to uninstall numpy and cause CI to fail.
+        python -m pip install --upgrade numpy
         python qlib/workflow/cli.py examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml
-        python -m pip uninstall -y pyqlib
-
-     # Test Qlib installed from source
-    - name: Install Qlib from source
-      run: |
-        pip install --upgrade cython jupyter jupyter_contrib_nbextensions numpy scipy scikit-learn # installing without this line will cause errors on GitHub Actions, while instsalling locally won't
-        pip install gym tianshou torch
-        pip install -e .
-
-    - name: Install test dependencies
-      run: |
-        pip install --upgrade pip
-        pip install black pytest
 
     - name: Unit tests with Pytest
       run: |
-        pip install -r scripts/data_collector/pit/requirements.txt
         cd tests
-        python -m pytest . --durations=10
-
-    - name: Test workflow by config (install from source)
-      run: |
-        python qlib/workflow/cli.py examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml
+        python -m pytest . -m "not slow" --durations=0

.github/workflows/test_qlib_from_source_slow.yml

@@ -0,0 +1,52 @@
+name: Test qlib from source slow
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+    timeout-minutes: 120
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [windows-latest, ubuntu-18.04, ubuntu-20.04, macos-11, macos-latest]
+        # not supporting 3.6 due to annotations is not supported https://stackoverflow.com/a/52890129
+        python-version: [3.7, 3.8]
+
+    steps:
+    - name: Test qlib from source slow
+      uses: actions/checkout@v2
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Set up Python tools
+      run: |
+        pip install --upgrade cython numpy pip
+        pip install -e .[dev]
+
+    - name: Downloads dependencies data
+      run: |
+        python scripts/get_data.py qlib_data --name qlib_data_simple --target_dir ~/.qlib/qlib_data/cn_data --interval 1d --region cn
+
+    - name: Install Lightgbm for MacOS
+      if: ${{ matrix.os == 'macos-11' || matrix.os == 'macos-latest' }}
+      run: |
+        /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Microsoft/qlib/main/.github/brew_install.sh)"
+        HOMEBREW_NO_AUTO_UPDATE=1 brew install lightgbm
+        # FIX MacOS error: Segmentation fault
+        # reference: https://github.com/microsoft/LightGBM/issues/4229
+        wget https://raw.githubusercontent.com/Homebrew/homebrew-core/fb8323f2b170bd4ae97e1bac9bf3e2983af3fdb0/Formula/libomp.rb
+        brew unlink libomp
+        brew install libomp.rb
+
+    - name: Unit tests with Pytest
+      run: |
+        cd tests
+        python -m pytest . -m "slow" --durations=0

qlib/data/storage/file_storage.py

@@ -106,10 +106,7 @@ class FileCalendarStorage(FileStorageMixin, CalendarStorage):
         if not self.uri.exists():
             self._write_calendar(values=[])
         with self.uri.open("rb") as fp:
-            return [
-                str(x)
-                for x in np.loadtxt(fp, str, skiprows=skip_rows, max_rows=n_rows, delimiter="\n", encoding="utf-8")
-            ]
+            return [str(x) for x in np.loadtxt(fp, str, skiprows=skip_rows, max_rows=n_rows, encoding="utf-8")]
 
     def _write_calendar(self, values: Iterable[CalVT], mode: str = "wb"):
         with self.uri.open(mode=mode) as fp:

setup.py

@@ -137,10 +137,24 @@ setup(
             "sphinx",
             "sphinx_rtd_theme",
             "pre-commit",
+            # CI dependencies
+            "wheel",
+            "setuptools",
+            "black",
+            "pylint",
+            "mypy",
+            "flake8",
+            "readthedocs_sphinx_ext",
+            "cmake",
+            "lxml",
+            "baostock",
+            "yahooquery",
+            "beautifulsoup4",
+            "tianshou",
+            "gym>=0.24",  # If you do not put gym at the end, gym will degrade causing pytest results to fail.
         ],
         "rl": [
             "tianshou",
-            "gym",
             "torch",
         ],
     },

tests/pytest.ini

@@ -1,4 +1,6 @@
 [pytest]
+markers =
+    slow: marks tests as slow (deselect with '-m "not slow"')
 filterwarnings =
     ignore:.*rng.randint:DeprecationWarning
     ignore:.*Casting input x to numpy array:UserWarning

tests/rolling_tests/test_update_pred.py

@@ -1,5 +1,6 @@
 import copy
 import unittest
+import pytest
 
 import fire
 import pandas as pd
@@ -14,6 +15,7 @@ from qlib.workflow.online.update import LabelUpdater
 
 
 class TestRolling(TestAutoData):
+    @pytest.mark.slow
     def test_update_pred(self):
         """
         This test is for testing if it will raise error if the `to_date` is out of the boundary.
@@ -73,6 +75,7 @@ class TestRolling(TestAutoData):
         # this range is fixed now
         self.assertTrue((updated_pred.loc[mod_range2] == -2).all().item())
 
+    @pytest.mark.slow
     def test_update_label(self):
 
         task = copy.deepcopy(CSI300_GBDT_TASK)

tests/test_all_pipeline.py

@@ -4,6 +4,7 @@
 import sys
 import shutil
 import unittest
+import pytest
 from pathlib import Path
 
 import qlib
@@ -184,16 +185,19 @@ class TestAllFlow(TestAutoData):
     def tearDownClass(cls) -> None:
         shutil.rmtree(cls.URI_PATH.lstrip("file:"))
 
+    @pytest.mark.slow
     def test_0_train_with_sigana(self):
         TestAllFlow.PRED_SCORE, ic_ric, uri_path = train_with_sigana(self.URI_PATH)
         self.assertGreaterEqual(ic_ric["ic"].all(), 0, "train failed")
         self.assertGreaterEqual(ic_ric["ric"].all(), 0, "train failed")
 
+    @pytest.mark.slow
     def test_1_train(self):
         TestAllFlow.PRED_SCORE, ic_ric, TestAllFlow.RID = train(self.URI_PATH)
         self.assertGreaterEqual(ic_ric["ic"].all(), 0, "train failed")
         self.assertGreaterEqual(ic_ric["ric"].all(), 0, "train failed")
 
+    @pytest.mark.slow
     def test_2_backtest(self):
         analyze_df = backtest_analysis(TestAllFlow.PRED_SCORE, TestAllFlow.RID, self.URI_PATH)
         self.assertGreaterEqual(
@@ -203,6 +207,7 @@ class TestAllFlow(TestAutoData):
         )
         self.assertTrue(not analyze_df.isna().any().any(), "backtest failed")
 
+    @pytest.mark.slow
     def test_3_expmanager(self):
         pass_default, pass_current, uri_path = fake_experiment()
         self.assertTrue(pass_default, msg="default uri is incorrect")

tests/test_contrib_workflow.py

@@ -4,6 +4,7 @@
 from qlib.workflow.record_temp import SignalRecord
 import shutil
 import unittest
+import pytest
 from pathlib import Path
 
 from qlib.contrib.workflow import MultiSegRecord, SignalMseRecord
@@ -47,9 +48,11 @@ class TestAllFlow(TestAutoData):
     def tearDownClass(cls) -> None:
         shutil.rmtree(cls.URI_PATH.lstrip("file:"))
 
+    @pytest.mark.slow
     def test_0_multiseg(self):
         uri_path = train_multiseg(self.URI_PATH)
 
+    @pytest.mark.slow
     def test_1_mse(self):
         uri_path = train_mse(self.URI_PATH)
 

tests/test_dataset.py

@@ -2,6 +2,7 @@
 # Licensed under the MIT License.
 
 import unittest
+import pytest
 import sys
 from qlib.tests import TestAutoData
 from qlib.data.dataset import TSDatasetH
@@ -11,6 +12,7 @@ from qlib.data.dataset.handler import DataHandlerLP
 
 
 class TestDataset(TestAutoData):
+    @pytest.mark.slow
     def testTSDataset(self):
         tsdh = TSDatasetH(
             handler={

tests/test_pit.py

@@ -6,12 +6,13 @@ import sys
 import qlib
 import shutil
 import unittest
+import pytest
 import pandas as pd
 import baostock as bs
 from pathlib import Path
 
 from qlib.data import D
-from scripts.get_data import GetData
+from qlib.tests.data import GetData
 from scripts.dump_pit import DumpPitData
 
 sys.path.append(str(Path(__file__).resolve().parent.parent.joinpath("scripts/data_collector/pit")))
@@ -119,6 +120,7 @@ class TestPIT(unittest.TestCase):
         """
         self.check_same(data, expect)
 
+    @pytest.mark.slow
     def test_expr(self):
         fields = [
             "P(Mean($$roewa_q, 1))",