From fd0d1d9967db738cd0485fec238684ff2ade5a64 Mon Sep 17 00:00:00 2001 From: J-shang <33053116+J-shang@users.noreply.github.com> Date: Tue, 14 Dec 2021 11:28:56 +0800 Subject: [PATCH] update interim patch (#4383) --- Dockerfile | 4 ++-- interim_vision_patch.py | 20 -------------------- pipelines/fast-test.yml | 22 ++++++++++++++++++++-- pipelines/integration-test-hybrid.yml | 4 ++-- pipelines/integration-test-remote-l2l.yml | 4 ++-- test/vso_tools/interim_patch.py | 17 +++++++++++++++++ 6 files changed, 43 insertions(+), 28 deletions(-) delete mode 100644 interim_vision_patch.py create mode 100644 test/vso_tools/interim_patch.py diff --git a/Dockerfile b/Dockerfile index aef1dadc4..df45803ef 100644 --- a/Dockerfile +++ b/Dockerfile @@ -80,8 +80,8 @@ RUN python3 -m pip install nni-${NNI_RELEASE}-py3-none-manylinux1_x86_64.whl # # Vision patch. Need del later # -COPY interim_vision_patch.py . -RUN python3 interim_vision_patch.py +COPY test/vso_tools/interim_patch.py . +RUN python3 interim_patch.py # # install aml package diff --git a/interim_vision_patch.py b/interim_vision_patch.py deleted file mode 100644 index 9377e05f8..000000000 --- a/interim_vision_patch.py +++ /dev/null @@ -1,20 +0,0 @@ -import os -import torchvision.datasets.mnist as mnist - -file_name = mnist.__file__ -dummy_file_name = os.path.join(os.path.dirname(file_name), 'mnist_dummy.py') -with open(file_name, 'r') as fr, open(dummy_file_name, 'w') as fw: - origin_text = fr.read() - mnist_head = origin_text.find('class MNIST(') - reasource_head = origin_text.find('resources = [', mnist_head) - reasource_tail = origin_text.find(']\n', reasource_head) - top = origin_text[:reasource_head] - reasource = "resources = [('https://ossci-datasets.s3.amazonaws.com/mnist/train-images-idx3-ubyte.gz', 'f68b3c2dcbeaaa9fbdd348bbdeb94873'),('https://ossci-datasets.s3.amazonaws.com/mnist/train-labels-idx1-ubyte.gz', 'd53e105ee54ea40749a09fcbcd1e9432'),('https://ossci-datasets.s3.amazonaws.com/mnist/t10k-images-idx3-ubyte.gz', '9fb629c4189551a2d022fa330f9573f3'),('https://ossci-datasets.s3.amazonaws.com/mnist/t10k-labels-idx1-ubyte.gz', 'ec29112dd5afa0611ce80d1b7f02629c')]\n" - bottom = origin_text[reasource_tail + 2:] - fw.write(top) - fw.write(reasource) - fw.write(bottom) - -if os.path.exists(dummy_file_name): - os.remove(file_name) - os.rename(dummy_file_name, file_name) diff --git a/pipelines/fast-test.yml b/pipelines/fast-test.yml index 112b8ad82..54b8c8286 100644 --- a/pipelines/fast-test.yml +++ b/pipelines/fast-test.yml @@ -179,6 +179,12 @@ stages: python -m pip install -e .[PPOTuner,DNGO] displayName: Install extra dependencies + # Need del later + - script: | + set -e + python test/vso_tools/interim_patch.py + displayName: Torch utils tensorboard interim patch + - script: | set -e cd test @@ -281,8 +287,8 @@ stages: # Need del later - script: | set -e - python interim_vision_patch.py - displayName: Vision MNIST Patch + python test/vso_tools/interim_patch.py + displayName: Torch utils tensorboard interim patch - script: | @@ -368,6 +374,12 @@ stages: python -m pip install -e .[SMAC,BOHB,PPOTuner,DNGO] displayName: Install extra dependencies + # Need del later + - script: | + set -e + python test/vso_tools/interim_patch.py + displayName: Torch utils tensorboard interim patch + - script: | cd test python -m pytest ut @@ -429,6 +441,12 @@ stages: python -m pip install -e .[DNGO] displayName: Install extra dependencies + # Need del later + - script: | + set -e + python test/vso_tools/interim_patch.py + displayName: Torch utils tensorboard interim patch + - script: | cd test python -m pytest ut diff --git a/pipelines/integration-test-hybrid.yml b/pipelines/integration-test-hybrid.yml index c65b7b739..b852ed1b2 100644 --- a/pipelines/integration-test-hybrid.yml +++ b/pipelines/integration-test-hybrid.yml @@ -62,10 +62,10 @@ jobs: - task: CopyFilesOverSSH@0 inputs: sshEndpoint: $(worker) - contents: interim_vision_patch.py + contents: test/vso_tools/interim_patch.py targetFolder: /tmp/nnitest/$(Build.BuildId) overwrite: true - displayName: Copy vision patch to remote machine + displayName: Copy torch patch to remote machine timeoutInMinutes: 10 - task: SSH@0 diff --git a/pipelines/integration-test-remote-l2l.yml b/pipelines/integration-test-remote-l2l.yml index e73319452..b76869e93 100644 --- a/pipelines/integration-test-remote-l2l.yml +++ b/pipelines/integration-test-remote-l2l.yml @@ -67,10 +67,10 @@ jobs: - task: CopyFilesOverSSH@0 inputs: sshEndpoint: $(worker) - contents: interim_vision_patch.py + contents: test/vso_tools/interim_patch.py targetFolder: /tmp/nnitest/$(Build.BuildId) overwrite: true - displayName: Copy vision patch to remote machine + displayName: Copy torch patch to remote machine timeoutInMinutes: 10 - task: SSH@0 diff --git a/test/vso_tools/interim_patch.py b/test/vso_tools/interim_patch.py new file mode 100644 index 000000000..83f3183e9 --- /dev/null +++ b/test/vso_tools/interim_patch.py @@ -0,0 +1,17 @@ +# fix setuptools.distutils import in torch +import os +from torch import utils + +file_name = os.path.join(os.path.dirname(utils.__file__), 'tensorboard/__init__.py') +dummy_file_name = os.path.join(os.path.dirname(file_name), '__dummy_init__.py') +if os.path.exists(file_name): + with open(file_name, 'r') as fr, open(dummy_file_name, 'w') as fw: + origin_text = fr.read() + patched_text = origin_text.replace('from setuptools import distutils', '', 1) + patched_text = patched_text.replace('LooseVersion = distutils.version.LooseVersion', 'from distutils.version import LooseVersion', 1) + patched_text = patched_text.replace('del distutils', '', 1) + fw.write(patched_text) + +if os.path.exists(dummy_file_name): + os.remove(file_name) + os.rename(dummy_file_name, file_name)