Update Python setup for require packages (#387)

__Description__

Update Python setup for require packages.

__Major Revisions__
* downgrade requests version to be compatible with python 3.6, add corresponding pipeline for 3.6
* add extra entry in extras_require for nested packages
* update `pip install` contents accordingly

.azure-pipelines/cpu-unit-test.yml

@@ -5,18 +5,29 @@ trigger:
   - main
   - release/*
 
+strategy:
+  matrix:
+    python-3.6:
+      imageTag: '3.6'
+    python-3.7:
+      imageTag: '3.7'
+    # TODO
+    #python-latest:
+    #  imageTag: '3'
+  maxParallel: 3
+
 pool:
   vmImage: ubuntu-latest
 
 container:
-  image: python:3.7
+  image: python:$(imageTag)
 
 steps:
   - script: |
       echo "##vso[task.prependpath]$HOME/.local/bin"
     displayName: Export path
   - script: |
-      python3 -m pip install .[test,torch]
+      python3 -m pip install .[test,cpuworker]
       make postinstall
     displayName: Install dependencies
   - script: |
@@ -29,7 +40,7 @@ steps:
     env:
       SB_TEST_CUDA: '0'
   - script: |
-      bash <(curl -s https://codecov.io/bash) -cF cpu-unit-test
+      bash <(curl -s https://codecov.io/bash) -cF cpu-python$(imageTag)-unit-test
     displayName: Report coverage results
     env:
       CODECOV_TOKEN: $(codecovToken)

.azure-pipelines/cuda-unit-test.yml

@@ -18,7 +18,7 @@ steps:
       echo "##vso[task.prependpath]$HOME/.local/bin"
     displayName: Export path
   - script: |
-      python3 -m pip install .[test,nvidia,torch,ort,mpi]
+      python3 -m pip install .[test,nvworker]
       make postinstall
     displayName: Install dependencies
   - script: |

.devcontainer/post-create-command.sh

@@ -4,7 +4,7 @@ set -e
 export PATH=${HOME}/.local/bin:${PATH}
 
 # install python packages
-pip install .[dev,test,torch] --user --no-cache-dir --progress-bar=off --use-feature=in-tree-build
+pip install .[develop,cpuworker] --user --no-cache-dir --progress-bar=off --use-feature=in-tree-build
 pre-commit install --install-hooks
 
 # install nodejs packages

dockerfile/cuda11.1.1.dockerfile

@@ -128,6 +128,6 @@ ADD third_party third_party
 RUN make -C third_party cuda
 
 ADD . .
-RUN python3 -m pip install .[nvidia,torch,ort,mpi] && \
+RUN python3 -m pip install .[nvworker] && \
     make cppbuild && \
     make postinstall

dockerfile/rocm5.0.x.dockerfile

@@ -124,6 +124,6 @@ ADD third_party third_party
 RUN make -C third_party rocm
 
 ADD . .
-RUN python3 -m pip install .[torch,ort,mpi]  && \
+RUN python3 -m pip install .[amdworker]  && \
     make cppbuild && \
     make postinstall

dockerfile/rocm5.1.x.dockerfile

@@ -139,6 +139,6 @@ ADD third_party third_party
 RUN make ROCBLAS_BRANCH=release/rocm-rel-5.1 -C third_party rocm
 
 ADD . .
-RUN python3 -m pip install .[torch,ort,mpi]  && \
+RUN python3 -m pip install .[amdworker]  && \
     make cppbuild && \
     make postinstall

docs/developer-guides/development.md

@@ -19,7 +19,7 @@ Follow [System Requirements](../getting-started/installation.mdx).
 git clone --recurse-submodules -j8 https://github.com/microsoft/superbenchmark
 cd superbenchmark
 
-python3 -m pip install -e .[dev,test]
+python3 -m pip install -e .[develop]
 ```
 
 ## Lint and Test

setup.py

@@ -152,11 +152,11 @@ setup(
         'natsort>=7.1.1',
         'networkx>=1.11',
         'numpy>=1.19.2',
-        'openpyxl>=3.0.7',
         'omegaconf==2.0.6',
+        'openpyxl>=3.0.7',
         'pandas>=1.1.5',
         'pyyaml>=5.3',
-        'requests>=2.28.1',
+        'requests>=2.27.1',
         'seaborn>=0.11.2',
         'tcping>=0.1.1rc1',
         'urllib3>=1.26.9',
@@ -164,35 +164,45 @@ setup(
         'xlsxwriter>=1.3.8',
         'xmltodict>=0.12.0',
     ],
-    extras_require={
+    extras_require=(
-        'dev': ['pre-commit>=2.10.0'],
+        lambda x: {
-        'test': [
+            **x,
-            'flake8-docstrings>=1.5.0',
+            'develop': x['dev'] + x['test'],
-            'flake8-quotes>=3.2.0',
+            'cpuworker': x['torch'],
-            'flake8>=3.8.4',
+            'amdworker': x['torch'] + x['ort'] + x['mpi'],
-            'mypy>=0.800',
+            'nvworker': x['torch'] + x['ort'] + x['mpi'] + x['nvidia'],
-            'pydocstyle>=5.1.1',
+        }
-            'pytest-cov>=2.11.1',
+    )(
-            'pytest-subtests>=0.4.0',
+        {
-            'pytest>=6.2.2',
+            'dev': ['pre-commit>=2.10.0'],
-            'types-markdown',
+            'test': [
-            'types-pkg_resources',
+                'flake8-docstrings>=1.5.0',
-            'types-pyyaml',
+                'flake8-quotes>=3.2.0',
-            'vcrpy>=4.1.1',
+                'flake8>=3.8.4',
-            'yapf==0.31.0',
+                'mypy>=0.800',
-        ],
+                'pydocstyle>=5.1.1',
-        'nvidia': ['py3nvml>=0.2.6'],
+                'pytest-cov>=2.11.1',
-        'ort': [
+                'pytest-subtests>=0.4.0',
-            'onnx>=1.10.2',
+                'pytest>=6.2.2',
-            'onnxruntime-gpu==1.10.0',
+                'types-markdown',
-        ],
+                'types-pkg_resources',
-        'torch': [
+                'types-pyyaml',
-            'torch>=1.7.0a0',
+                'vcrpy>=4.1.1',
-            'torchvision>=0.8.0a0',
+                'yapf==0.31.0',
-            'transformers>=4.3.3',
+            ],
-        ],
+            'torch': [
-        'mpi': ['mpi4py>=3.1.3'],
+                'torch>=1.7.0a0',
-    },
+                'torchvision>=0.8.0a0',
+                'transformers>=4.3.3',
+            ],
+            'ort': [
+                'onnx>=1.10.2',
+                'onnxruntime-gpu==1.10.0',
+            ],
+            'mpi': ['mpi4py>=3.1.3'],
+            'nvidia': ['py3nvml>=0.2.6'],
+        }
+    ),
     include_package_data=True,
     entry_points={
         'console_scripts': [