From 0820b2b1319a0b5ef32b50f314dae9738b9d286f Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 22 Sep 2021 09:41:00 -0500 Subject: [PATCH] Reduce number of files listing deps (#154) * Reduce number of files listing deps * Install cython before anything else * Fix missing deps before install * Fix typo in setup.cfg extra option * Add Read the Docs config file * nbmake only used for testing docs, not building docs --- .github/workflows/docs.yaml | 10 +++--- .github/workflows/release.yaml | 10 ++---- .github/workflows/style.yaml | 12 +++---- .github/workflows/tests.yaml | 7 ++-- .readthedocs.yaml | 19 ++++++++++ docs/requirements.txt | 62 ++------------------------------ environment.yml | 6 ++-- requirements.txt | 66 ---------------------------------- setup.cfg | 62 +++++++++++++++++++++----------- spack.yaml | 40 --------------------- 10 files changed, 82 insertions(+), 212 deletions(-) create mode 100644 .readthedocs.yaml delete mode 100644 requirements.txt delete mode 100644 spack.yaml diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index b0d1c02df..0669a1323 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -1,3 +1,5 @@ +# TODO: it should be possible to test documentation builds directly on Read the Docs: +# https://docs.readthedocs.io/en/stable/pull-requests.html name: docs on: push: @@ -17,8 +19,8 @@ jobs: uses: actions/setup-python@v2 with: python-version: 3.9 - - name: Install pydocstyle - run: pip install 'pydocstyle[toml]>=6.1' + - name: Install pip dependencies + run: pip install .[style] - name: Run pydocstyle checks run: pydocstyle sphinx: @@ -35,7 +37,7 @@ jobs: run: sudo apt-get install pandoc - name: Install pip dependencies run: | - pip install cython numpy # needed for pycocotools - pip install -r requirements.txt + pip install .[train] + pip install -r docs/requirements.txt - name: Run sphinx checks run: cd docs && make html diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index d560e897b..a3ed9b83c 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -18,10 +18,7 @@ jobs: with: python-version: 3.9 - name: Install pip dependencies - run: | - pip install cython numpy # needed for pycocotools - pip install -r requirements.txt - pip list + run: pip install .[tests] - name: Run notebook checks run: pytest --nbmake docs/tutorials integration: @@ -35,9 +32,6 @@ jobs: with: python-version: 3.9 - name: Install pip dependencies - run: | - pip install cython numpy # needed for pycocotools - pip install -r requirements.txt - pip list + run: pip install .[tests] - name: Run integration checks run: pytest -m slow diff --git a/.github/workflows/style.yaml b/.github/workflows/style.yaml index 73a14fecd..b76fd81d2 100644 --- a/.github/workflows/style.yaml +++ b/.github/workflows/style.yaml @@ -17,8 +17,8 @@ jobs: uses: actions/setup-python@v2 with: python-version: 3.9 - - name: Install black - run: pip install 'black>=21.4b0' + - name: Install pip dependencies + run: pip install .[style] - name: Run black checks run: black . --check --diff flake8: @@ -31,8 +31,8 @@ jobs: uses: actions/setup-python@v2 with: python-version: 3.9 - - name: Install flake8 - run: pip install 'flake8>=3.8' + - name: Install pip dependencies + run: pip install .[style] - name: Run flake8 checks run: flake8 isort: @@ -45,7 +45,7 @@ jobs: uses: actions/setup-python@v2 with: python-version: 3.9 - - name: Install isort - run: pip install 'isort[colors]>=5.8' + - name: Install pip dependencies + run: pip install .[style] - name: Run isort checks run: isort . --check --diff diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 126902b2b..6c208c48e 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -17,10 +17,10 @@ jobs: uses: actions/setup-python@v2 with: python-version: 3.9 - - name: Install dependencies + - name: Install pip dependencies run: | pip install cython numpy # needed for pycocotools - pip install -r requirements.txt + pip install .[datasets,tests,train] - name: Run mypy checks run: mypy . pytest: @@ -62,8 +62,7 @@ jobs: - name: Install pip dependencies run: | pip install cython numpy # needed for pycocotools - pip install -r requirements.txt - pip list + pip install .[datasets,tests,train] - name: Run pytest checks run: pytest --cov=torchgeo --cov-report=xml - name: Report coverage diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 000000000..c67467745 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,19 @@ +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Configuration of the Python environment to be used +python: + install: + - requirements: docs/requirements.txt + - method: pip + path: . + extra_requirements: + - train + +# Configuration for Sphinx documentation +sphinx: + configuration: docs/conf.py + fail_on_warning: true diff --git a/docs/requirements.txt b/docs/requirements.txt index da5053ee7..c4a5e359b 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,66 +1,8 @@ -# black 21+ required for Python 3.9 support -black>=21.4b0 -einops -# fiona 1.5+ required for fiona.transform module -fiona>=1.5 -# flake8 3.8+ depends on pyflakes 2.2+, which fixes a bug with mypy error code ignores: -# https://github.com/PyCQA/pyflakes/pull/455 -flake8>=3.8 -h5py # ipywidgets 7+ required for nbsphinx ipywidgets>=7 -# isort 5.8+ required for extend_skip option -isort[colors]>=5.8 -jupyterlab -matplotlib -# mypy 0.900+ required for pyproject.toml support -mypy>=0.900 -# nbmake 0.1+ required to fix path_source bug -nbmake>=0.1 # nbsphinx 0.8.5 fixes bug with nbformat attributes nbsphinx>=0.8.5 -numpy -# omegaconf 2.1+ required for to_object method -omegaconf>=2.1 -opencv-python -# pillow 2.9+ required for height attribute -pillow>=2.9 -pycocotools -# pydocstyle 6.1+ required for pyproject.toml support -pydocstyle[toml]>=6.1 -# pyproj 2.2.0+ required for CRS object, note: pyproj 3.1+ contains threadsafe Transformer and CRS implementations -pyproj>=2.2.0 -# pytest 6+ required for pyproject.toml support -pytest>=6 -# pytest-cov 2.4+ required for pytest --cov flags -pytest-cov>=2.4 -# pytorch-lightning 1.3+ required for gradient_clip_algorithm argument to Trainer -pytorch-lightning>=1.3 # release versions missing files, must install from master --e git+https://github.com/pytorch/pytorch_sphinx_theme#egg=pytorch_sphinx_theme -# radiant-mlhub 0.2.1+ required for api_key bugfix: -# https://github.com/radiantearth/radiant-mlhub/pull/48 -radiant-mlhub>=0.2.1 -# rarfile 3+ required for correct Rar file detection -rarfile>=3 -# rasterio 1.0.16+ required for CRS support -rasterio>=1.0.16 -# rtree 0.5+ required for 3D index support -rtree>=0.5 -# scikit-learn 0.18+ required for sklearn.model_selection module -scikit-learn>=0.18 -# scipy 0.9.0+ required for scipy.io.wavfile.read -scipy>=0.9.0 -# segmentation-models-pytorch 0.2+ required for smp.losses module -segmentation-models-pytorch>=0.2 -# setuptools 30.4+ required for options.packages.find section in setup.cfg -setuptools>=30.4 -# shapely 1.3.0+ is required for Python 3.x -shapely>=1.3.0 +-e git+https://github.com/pytorch/pytorch_sphinx_theme.git#egg=pytorch_sphinx_theme # sphinx 3+ required for autodoc_typehints = description -sphinx>=3,!=4.1.2 -# torch 1.7+ required for typing -torch>=1.7 -torchmetrics -# torchvision 0.3+ required for download_file_from_google_drive -torchvision>=0.3 +sphinx>=3 diff --git a/environment.yml b/environment.yml index f24e8ba6b..4b17e8c58 100644 --- a/environment.yml +++ b/environment.yml @@ -9,12 +9,12 @@ dependencies: - numpy - pip - pycocotools - - pyproj>=2.2.0 + - pyproj>=2.2 - python>=3.6 - pytorch-gpu>=1.7 - rarfile>=3 - rasterio>=1.0.16 - - shapely>=1.3.0 + - shapely>=1.3 - torchvision>=0.3 - pip: - black>=21.4b0 @@ -36,7 +36,7 @@ dependencies: - radiant-mlhub>=0.2.1 - rtree>=0.5 - scikit-learn>=0.18 - - scipy>=0.9.0 + - scipy>=0.9 - segmentation-models-pytorch>=0.2 - setuptools>=30.4 - sphinx>=3 diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index a1d460e88..000000000 --- a/requirements.txt +++ /dev/null @@ -1,66 +0,0 @@ -# black 21+ required for Python 3.9 support -black>=21.4b0 -einops -# fiona 1.5+ required for fiona.transform module -fiona>=1.5 -# flake8 3.8+ depends on pyflakes 2.2+, which fixes a bug with mypy error code ignores: -# https://github.com/PyCQA/pyflakes/pull/455 -flake8>=3.8 -h5py -# ipywidgets 7+ required for nbsphinx -ipywidgets>=7 -# isort 5.8+ required for extend_skip option -isort[colors]>=5.8 -jupyterlab -matplotlib -# mypy 0.900+ required for pyproject.toml support -mypy>=0.900 -# nbmake 0.1+ required to fix path_source bug -nbmake>=0.1 -# nbsphinx 0.8.5 fixes bug with nbformat attributes -nbsphinx>=0.8.5 -numpy -# omegaconf 2.1+ required for to_object method -omegaconf>=2.1 -opencv-python -# pillow 2.9+ required for height attribute -pillow>=2.9 -pycocotools -# pydocstyle 6.1+ required for pyproject.toml support -pydocstyle[toml]>=6.1 -# pyproj 2.2.0+ required for CRS object, note: pyproj 3.1+ contains threadsafe Transformer and CRS implementations -pyproj>=2.2.0 -# pytest 6+ required for pyproject.toml support -pytest>=6 -# pytest-cov 2.4+ required for pytest --cov flags -pytest-cov>=2.4 -# pytorch-lightning 1.3+ required for gradient_clip_algorithm argument to Trainer -pytorch-lightning>=1.3 -# release versions missing files, must install from master -git+https://github.com/pytorch/pytorch_sphinx_theme#egg=pytorch_sphinx_theme -# radiant-mlhub 0.2.1+ required for api_key bugfix: -# https://github.com/radiantearth/radiant-mlhub/pull/48 -radiant-mlhub>=0.2.1 -# rarfile 3+ required for correct Rar file detection -rarfile>=3 -# rasterio 1.0.16+ required for CRS support -rasterio>=1.0.16 -# rtree 0.5+ required for 3D index support -rtree>=0.5 -# scikit-learn 0.18+ required for sklearn.model_selection module -scikit-learn>=0.18 -# scipy 0.9.0+ required for scipy.io.wavfile.read -scipy>=0.9.0 -# segmentation-models-pytorch 0.2+ required for smp.losses module -segmentation-models-pytorch>=0.2 -# setuptools 30.4+ required for options.packages.find section in setup.cfg -setuptools>=30.4 -# shapely 1.3.0+ is required for Python 3.x -shapely>=1.3.0 -# sphinx 3+ required for autodoc_typehints = description -sphinx>=3 -# torch 1.7+ required for typing -torch>=1.7 -torchmetrics -# torchvision 0.3+ required for download_file_from_google_drive -torchvision>=0.3 diff --git a/setup.cfg b/setup.cfg index 909dc565d..52b390cb8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,18 +24,27 @@ keywords = pytorch, deep learning, machine learning [options] setup_requires = + # setuptools 30.4+ required for options.packages.find section in setup.cfg setuptools>=30.4 install_requires = einops + # fiona 1.5+ required for fiona.transform module fiona>=1.5 matplotlib numpy + # pillow 2.9+ required for height attribute pillow>=2.9 - pyproj>=2.2.0 + # pyproj 2.2+ required for CRS object + pyproj>=2.2 + # rasterio 1.0.16+ required for CRS support rasterio>=1.0.16 + # rtree 0.5+ required for 3D index support rtree>=0.5 - shapely>=1.3.0 + # shapely 1.3+ required for Python 3 support + shapely>=1.3 + # torch 1.7+ required for typing torch>=1.7 + # torchvision 0.3+ required for download_file_from_google_drive torchvision>=0.3 python_requires = >= 3.6 packages = find: @@ -49,30 +58,41 @@ datasets = h5py opencv-python pycocotools + # radiant-mlhub 0.2.1+ required for api_key bugfix: + # https://github.com/radiantearth/radiant-mlhub/pull/48 radiant-mlhub>=0.2.1 + # rarfile 3+ required for correct Rar file detection rarfile>=3 - scipy>=0.9.0 - -# Optional developer requirements -docs = - ipywidgets>=7 - jupyterlab - nbmake>=0.1 - nbsphinx>=0.8.5 - sphinx>=3 - pydocstyle[toml]>=6.1 - pytorch-sphinx-theme -style = - black>=21.4b0 - flake8>=3.8 - isort[colors]>=5.8 -tests = - mypy>=0.900 - pytest>=6 - pytest-cov>=2.4 + # scipy 0.9+ required for scipy.io.wavfile.read + scipy>=0.9 +# Optional trainer requirements train = + # omegaconf 2.1+ required for to_object method omegaconf>=2.1 + # pytorch-lightning 1.3+ required for gradient_clip_algorithm argument to Trainer pytorch-lightning>=1.3 + # scikit-learn 0.18+ required for sklearn.model_selection module scikit-learn>=0.18 + # segmentation-models-pytorch 0.2+ required for smp.losses module segmentation-models-pytorch>=0.2 torchmetrics +# Optional developer requirements +style = + # black 21+ required for Python 3.9 support + black>=21.4b0 + # flake8 3.8+ depends on pyflakes 2.2+, which fixes a bug with mypy error code ignores: + # https://github.com/PyCQA/pyflakes/pull/455 + flake8>=3.8 + # isort 5.8+ required for extend_skip option + isort[colors]>=5.8 + # pydocstyle 6.1+ required for pyproject.toml support + pydocstyle[toml]>=6.1 +tests = + # mypy 0.900+ required for pyproject.toml support + mypy>=0.900 + # nbmake 0.1+ required to fix path_source bug + nbmake>=0.1 + # pytest 6+ required for pyproject.toml support + pytest>=6 + # pytest-cov 2.4+ required for pytest --cov flags + pytest-cov>=2.4 diff --git a/spack.yaml b/spack.yaml deleted file mode 100644 index 915fc4220..000000000 --- a/spack.yaml +++ /dev/null @@ -1,40 +0,0 @@ -spack: - specs: - - "opencv+python3+imgcodecs+tiff+jpeg+png" - - "pil@2.9:" - - "python@3.6:+bz2" - - "py-black@21:" - - "py-einops" - - "py-fiona@1.5:" - - "py-flake8@3.8:" - - "py-h5py" - - "py-ipywidgets@7:" - - "py-isort@5.8:+colors" - - "py-jupyterlab" - - "py-matplotlib" - - "py-mypy@0.900:" - - "py-nbmake@0.1:" - - "py-nbsphinx@0.8.5:" - - "py-numpy" - - "py-omegaconf@2.1:" - - "py-pycocotools" - - "py-pydocstyle@6.1:+toml" - - "py-pyproj@2.2.0:" - - "py-pytest@6:" - - "py-pytest-cov@2.4:" - - "py-pytorch-lightning@1.3:" - - "py-pytorch-sphinx-theme" - - "py-radiant-mlhub@0.2.1:" - - "py-rarfile@3:" - - "py-rasterio@1.0.16:" - - "py-rtree@0.5:" - - "py-scikit-learn@0.18:" - - "py-scipy@0.9.0:" - - "py-segmentation-models-pytorch@0.2:" - - "py-setuptools@30.4:" - - "py-shapely@1.3.0:" - - "py-sphinx@3:" - - "py-torch@1.7:" - - "py-torchmetrics" - - "py-torchvision@0.3:" - concretization: together