update interim patch (#4383)

Dockerfile

@@ -80,8 +80,8 @@ RUN python3 -m pip install nni-${NNI_RELEASE}-py3-none-manylinux1_x86_64.whl
 # 
 # Vision patch. Need del later
 # 
-COPY interim_vision_patch.py .
-RUN python3 interim_vision_patch.py
+COPY test/vso_tools/interim_patch.py .
+RUN python3 interim_patch.py
 
 #
 # install aml package

interim_vision_patch.py

@@ -1,20 +0,0 @@
-import os
-import torchvision.datasets.mnist as mnist
-
-file_name = mnist.__file__
-dummy_file_name = os.path.join(os.path.dirname(file_name), 'mnist_dummy.py')
-with open(file_name, 'r') as fr, open(dummy_file_name, 'w') as fw:
-    origin_text = fr.read()
-    mnist_head = origin_text.find('class MNIST(')
-    reasource_head = origin_text.find('resources = [', mnist_head)
-    reasource_tail = origin_text.find(']\n', reasource_head)
-    top = origin_text[:reasource_head]
-    reasource = "resources = [('https://ossci-datasets.s3.amazonaws.com/mnist/train-images-idx3-ubyte.gz', 'f68b3c2dcbeaaa9fbdd348bbdeb94873'),('https://ossci-datasets.s3.amazonaws.com/mnist/train-labels-idx1-ubyte.gz', 'd53e105ee54ea40749a09fcbcd1e9432'),('https://ossci-datasets.s3.amazonaws.com/mnist/t10k-images-idx3-ubyte.gz', '9fb629c4189551a2d022fa330f9573f3'),('https://ossci-datasets.s3.amazonaws.com/mnist/t10k-labels-idx1-ubyte.gz', 'ec29112dd5afa0611ce80d1b7f02629c')]\n"
-    bottom = origin_text[reasource_tail + 2:]
-    fw.write(top)
-    fw.write(reasource)
-    fw.write(bottom)
-
-if os.path.exists(dummy_file_name):
-    os.remove(file_name)
-    os.rename(dummy_file_name, file_name)

pipelines/fast-test.yml

@@ -179,6 +179,12 @@ stages:
         python -m pip install -e .[PPOTuner,DNGO]
       displayName: Install extra dependencies
 
+    # Need del later
+    - script: |
+        set -e
+        python test/vso_tools/interim_patch.py
+      displayName: Torch utils tensorboard interim patch
+
     - script: |
         set -e
         cd test
@@ -281,8 +287,8 @@ stages:
     # Need del later
     - script: |
         set -e
-        python interim_vision_patch.py
-      displayName: Vision MNIST Patch
+        python test/vso_tools/interim_patch.py
+      displayName: Torch utils tensorboard interim patch
 
 
     - script: |
@@ -368,6 +374,12 @@ stages:
         python -m pip install -e .[SMAC,BOHB,PPOTuner,DNGO]
       displayName: Install extra dependencies
 
+    # Need del later
+    - script: |
+        set -e
+        python test/vso_tools/interim_patch.py
+      displayName: Torch utils tensorboard interim patch
+
     - script: |
         cd test
         python -m pytest ut
@@ -429,6 +441,12 @@ stages:
         python -m pip install -e .[DNGO]
       displayName: Install extra dependencies
 
+    # Need del later
+    - script: |
+        set -e
+        python test/vso_tools/interim_patch.py
+      displayName: Torch utils tensorboard interim patch
+
     - script: |
         cd test
         python -m pytest ut

pipelines/integration-test-hybrid.yml

@@ -62,10 +62,10 @@ jobs:
   - task: CopyFilesOverSSH@0
     inputs:
       sshEndpoint: $(worker)
-      contents: interim_vision_patch.py
+      contents: test/vso_tools/interim_patch.py
       targetFolder: /tmp/nnitest/$(Build.BuildId)
       overwrite: true
-    displayName: Copy vision patch to remote machine
+    displayName: Copy torch patch to remote machine
     timeoutInMinutes: 10
 
   - task: SSH@0

pipelines/integration-test-remote-l2l.yml

@@ -67,10 +67,10 @@ jobs:
   - task: CopyFilesOverSSH@0
     inputs:
       sshEndpoint: $(worker)
-      contents: interim_vision_patch.py
+      contents: test/vso_tools/interim_patch.py
       targetFolder: /tmp/nnitest/$(Build.BuildId)
       overwrite: true
-    displayName: Copy vision patch to remote machine
+    displayName: Copy torch patch to remote machine
     timeoutInMinutes: 10
 
   - task: SSH@0

test/vso_tools/interim_patch.py

@@ -0,0 +1,17 @@
+# fix setuptools.distutils import in torch
+import os
+from torch import utils
+
+file_name = os.path.join(os.path.dirname(utils.__file__), 'tensorboard/__init__.py')
+dummy_file_name = os.path.join(os.path.dirname(file_name), '__dummy_init__.py')
+if os.path.exists(file_name):
+    with open(file_name, 'r') as fr, open(dummy_file_name, 'w') as fw:
+        origin_text = fr.read()
+        patched_text = origin_text.replace('from setuptools import distutils', '', 1)
+        patched_text = patched_text.replace('LooseVersion = distutils.version.LooseVersion', 'from distutils.version import LooseVersion', 1)
+        patched_text = patched_text.replace('del distutils', '', 1)
+        fw.write(patched_text)
+
+if os.path.exists(dummy_file_name):
+    os.remove(file_name)
+    os.rename(dummy_file_name, file_name)