Use python 3.9 on pipeline (#3881)

dependencies/recommended.txt

@@ -1,12 +1,13 @@
 # Recommended because some non-commonly-used modules/examples depend on those packages.
 
 -f https://download.pytorch.org/whl/torch_stable.html
-tensorflow
-keras
-torch == 1.6.0+cpu ; sys_platform != "darwin"
-torch == 1.6.0 ; sys_platform == "darwin"
-torchvision == 0.7.0+cpu ; sys_platform != "darwin"
-torchvision == 0.7.0 ; sys_platform == "darwin"
+tensorflow == 2.5.0
+keras == 2.4.3
+tensorboard == 2.5.0
+torch == 1.9.0+cpu ; sys_platform != "darwin"
+torch == 1.9.0 ; sys_platform == "darwin"
+torchvision == 0.10.0+cpu ; sys_platform != "darwin"
+torchvision == 0.10.0 ; sys_platform == "darwin"
 pytorch-lightning >= 1.1.1
 onnx
 peewee

dependencies/recommended_gpu.txt

@@ -2,9 +2,9 @@
 
 -f https://download.pytorch.org/whl/torch_stable.html
 tensorflow
-keras
-torch == 1.6.0
-torchvision == 0.7.0
+keras == 2.4.3
+torch == 1.9.0+cu111
+torchvision == 0.10.0+cu111
 pytorch-lightning >= 1.1.1
 onnx
 peewee

dependencies/recommended_legacy.txt

@@ -1,7 +1,7 @@
 -f https://download.pytorch.org/whl/torch_stable.html
 tensorflow == 1.15.4
-torch == 1.5.1+cpu
-torchvision == 0.6.1+cpu
+torch == 1.6.0+cpu
+torchvision == 0.7.0+cpu
 
 # It will install pytorch-lightning 0.8.x and unit tests won't work.
 # Latest version has conflict with tensorboard and tensorflow 1.x.

docs/en_US/NAS/QuickStart.rst

@@ -158,8 +158,6 @@ If the built-in model evaluators do not meet your requirement, or you already wr
 
 .. warning:: Mutations on the parameters of model evaluator (known as hyper-parameter tuning) is currently not supported but will be supported in the future.
 
-.. warning:: To use PyTorch-lightning with Retiarii, currently you need to install PyTorch-lightning v1.1.x (v1.2 is not supported).
-
 Launch an Experiment
 --------------------
 

examples/nas/oneshot/darts/utils.py

@@ -16,6 +16,6 @@ def accuracy(output, target, topk=(1,)):
 
     res = dict()
     for k in topk:
-        correct_k = correct[:k].view(-1).float().sum(0)
+        correct_k = correct[:k].reshape(-1).float().sum(0)
         res["acc{}".format(k)] = correct_k.mul_(1.0 / batch_size).item()
     return res

examples/nas/oneshot/enas/utils.py

@@ -19,7 +19,7 @@ def accuracy(output, target, topk=(1,)):
 
     res = dict()
     for k in topk:
-        correct_k = correct[:k].view(-1).float().sum(0)
+        correct_k = correct[:k].reshape(-1).float().sum(0)
         res["acc{}".format(k)] = correct_k.mul_(1.0 / batch_size).item()
     return res
 

examples/nas/oneshot/proxylessnas/retrain.py

@@ -26,7 +26,7 @@ def accuracy(output, target, topk=(1,)):
 
     res = []
     for k in topk:
-        correct_k = correct[:k].view(-1).float().sum(0, keepdim=True)
+        correct_k = correct[:k].reshape(-1).float().sum(0, keepdim=True)
         res.append(correct_k.mul_(100.0 / batch_size))
     return res
 

nni/algorithms/hpo/dngo_tuner.py

@@ -69,8 +69,18 @@ class DNGOTuner(Tuner):
             # random samples and pick best with model
             candidate_x = [_random_config(self.searchspace_json, self.random_state) for _ in range(self.sample_size)]
 
+            # The model has NaN issue when all the candidates are same
+            # Also we can save the predict time when this happens
+            if all(x == candidate_x[0] for x in candidate_x):
+                return candidate_x[0]
+
             x_test = np.array([np.array(list(xi.values())) for xi in candidate_x])
             m, v = self.model.predict(x_test)
+
+            # The model has NaN issue when all the candidates are very close
+            if np.isnan(m).any() or np.isnan(v).any():
+                return candidate_x[0]
+
             mean = torch.Tensor(m)
             sigma = torch.Tensor(v)
             u = (mean - torch.Tensor([0.95]).expand_as(mean)) / sigma

nni/algorithms/nas/pytorch/proxylessnas/utils.py

@@ -73,6 +73,6 @@ def accuracy(output, target, topk=(1,)):
 
     res = []
     for k in topk:
-        correct_k = correct[:k].view(-1).float().sum(0, keepdim=True)
+        correct_k = correct[:k].reshape(-1).float().sum(0, keepdim=True)
         res.append(correct_k.mul_(100.0 / batch_size))
     return res

nni/algorithms/nas/tensorflow/enas/trainer.py

@@ -1,6 +1,8 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT license.
 
+# pylint: skip-file
+
 import logging
 
 import tensorflow as tf

pipelines/fast-test.yml

@@ -124,7 +124,7 @@ stages:
     steps:
     - task: UsePythonVersion@0
       inputs:
-        versionSpec: 3.8
+        versionSpec: 3.9
       displayName: Configure Python version
 
     - task: NodeTool@0
@@ -170,15 +170,9 @@ stages:
     - script: |
         set -e
         python -m pip install -r dependencies/recommended.txt
-        python -m pip install -e .[SMAC,BOHB,PPOTuner,DNGO]
+        python -m pip install -e .[PPOTuner,DNGO]
       displayName: Install extra dependencies
 
-    # Need del later
-    - script: |
-        set -e
-        python interim_vision_patch.py
-      displayName: Vision MNIST Patch
-
     - script: |
         set -e
         cd test
@@ -368,12 +362,6 @@ stages:
         python -m pip install -e .[SMAC,BOHB,PPOTuner,DNGO]
       displayName: Install extra dependencies
 
-    # Need del later
-    - script: |
-        set -e
-        python interim_vision_patch.py
-      displayName: Vision MNIST Patch
-
     - script: |
         cd test
         python -m pytest ut
@@ -434,12 +422,6 @@ stages:
         python -m pip install -e .[DNGO]
       displayName: Install extra dependencies
 
-    # Need del later
-    - script: |
-        set -e
-        python interim_vision_patch.py
-      displayName: Vision MNIST Patch
-
     - script: |
         cd test
         python -m pytest ut

pipelines/full-test-linux.yml

@@ -33,12 +33,6 @@ jobs:
       python3 -m pip install -e .[SMAC,BOHB,PPOTuner,DNGO]
     displayName: Install extra dependencies
 
-  # Need del later
-  - script: |
-      set -e
-      python3 interim_vision_patch.py
-    displayName: Vision MNIST Patch
-
   - script: |
       set -e
       cd examples/tuners/customized_tuner

pipelines/full-test-windows.yml

@@ -29,12 +29,6 @@ jobs:
       python -m pip install -e .[DNGO]
     displayName: Install extra dependencies
 
-  # Need del later
-  - script: |
-      set -e
-      python interim_vision_patch.py
-    displayName: Vision MNIST Patch
-
   - script: |
       cd examples/tuners/customized_tuner
       python setup.py develop --user

test/ut/sdk/test_builtin_tuners.py

@@ -22,8 +22,10 @@ from nni.algorithms.hpo.pbt_tuner import PBTTuner
 from nni.algorithms.hpo.regularized_evolution_tuner import RegularizedEvolutionTuner
 from nni.runtime.msg_dispatcher import _pack_parameter, MsgDispatcher
 
-if sys.platform != 'win32':
+smac_imported = False
+if sys.platform != 'win32' and sys.version_info < (3, 9):
     from nni.algorithms.hpo.smac_tuner import SMACTuner
+    smac_imported = True
 
 from nni.tuner import Tuner
 
@@ -334,7 +336,7 @@ class BuiltinTunersTestCase(TestCase):
         self.import_data_test(tuner_fn)
 
     def test_smac(self):
-        if sys.platform == "win32":
+        if not smac_imported:
             return  # smac doesn't work on windows
         tuner_fn = lambda: SMACTuner()
         self.search_space_test_all(tuner_fn,