[pipeline] More aggressive cache (#4495)

dependencies/develop.txt

@@ -1,13 +1,14 @@
-pylint
+coverage
+cython
 flake8
+ipython
+jupyterlab
+nbsphinx
+pylint
+pytest
+pytest-azurepipelines
+pytest-cov
 sphinx
 sphinx-argparse
 sphinx-rtd-theme
 sphinxcontrib-websupport
-nbsphinx
-pytest
-pytest-cov
-pytest-azurepipelines
-coverage
-ipython
-jupyterlab

dependencies/recommended_legacy.txt

@@ -4,7 +4,7 @@ torchvision == 0.8.2+cpu
 
 # It will install pytorch-lightning 0.8.x and unit tests won't work.
 # Latest version has conflict with tensorboard and tensorflow 1.x.
-#pytorch-lightning
+pytorch-lightning
 torchmetrics
 
 onnx

dependencies/required_extra.txt

@@ -7,7 +7,7 @@ smac4nni
 
 # BOHB
 ConfigSpace>=0.4.17
-statsmodels==0.12.0
+statsmodels>=0.12.0
 
 # PPOTuner
 gym

pipelines/cache-dependencies.yml

@@ -0,0 +1,40 @@
+trigger: none
+pr: none
+schedules:
+- cron: "0 16 * * *"
+  branches:
+    include: [ master ]
+  always: true
+
+jobs:
+- job: ubuntu_latest
+  pool:
+    vmImage: ubuntu-latest
+  steps:
+  - template: templates/cache-dependencies-template.yml
+    parameters:
+      platform: ubuntu-latest
+
+- job: ubuntu_legacy
+  pool:
+    vmImage: ubuntu-18.04
+  steps:
+  - template: templates/cache-dependencies-template.yml
+    parameters:
+      platform: ubuntu-legacy
+
+- job: windows
+  pool:
+    vmImage: windows-latest
+  steps:
+  - template: templates/cache-dependencies-template.yml
+    parameters:
+      platform: windows
+
+- job: macos
+  pool:
+    vmImage: macOS-latest
+  steps:
+  - template: templates/cache-dependencies-template.yml
+    parameters:
+      platform: macos

pipelines/fast-test.yml

@@ -1,5 +1,6 @@
-# To reduce debug cost, steps are sorted differently on each platform,
-# so that a bug in any module will cause at least one platform to fail quickly.
+trigger:
+  branches:
+    exclude: [ l10n_master ]
 
 stages:
 - stage: lint
@@ -7,42 +8,20 @@ stages:
   - job: docs
     pool:
       vmImage: ubuntu-latest
-    variables:
-      PIP_CACHE_DIR: $(Pipeline.Workspace)/.pip
     steps:
-    - task: UsePythonVersion@0
-      inputs:
-        versionSpec: 3.9
-      displayName: Configure Python version
+    - template: templates/install-dependencies.yml
+      parameters:
+        platform: ubuntu-latest
+
     - script: |
         sudo apt-get install -y pandoc
-        sudo apt-get remove swig -y
-        sudo apt-get install swig3.0 -y
-        sudo ln -s /usr/bin/swig3.0 /usr/bin/swig
-      displayName: Install apt packages
-    - task: Cache@2
-      inputs:
-        key: 'python | "$(Agent.OS)" | dependencies/*.txt'
-        restoreKeys: | 
-          python | "$(Agent.OS)"
-          python
-        path: $(PIP_CACHE_DIR)
-      displayName: Cache pip packages
-    - script: |
-        set -e
-        python -m pip install -U -r dependencies/setup.txt
-        python -m pip install -r dependencies/develop.txt
-        python -m pip install -r dependencies/required.txt
-        python -m pip install -r dependencies/recommended.txt
-        python -m pip install -r dependencies/required_extra.txt
-      displayName: Install requirements
-    - script: |
-        python test/vso_tools/interim_patch.py
-      displayName: Apply patch
+      displayName: Install pandoc
+
     - script: |
         cd docs
         python tools/chineselink.py check
       displayName: Translation up-to-date
+
     - script: |
         cd docs/en_US
         sphinx-build -M html . _build -W --keep-going -T
@@ -51,37 +30,20 @@ stages:
   - job: python
     pool:
       vmImage: ubuntu-latest
-    variables:
-      PIP_CACHE_DIR: $(Pipeline.Workspace)/.pip
     steps:
-    - task: UsePythonVersion@0
-      inputs:
-        versionSpec: 3.9
-      displayName: Configure Python version
+    - template: templates/install-dependencies.yml
+      parameters:
+        platform: ubuntu-latest
+
     - script: |
-        sudo apt-get remove swig -y
-        sudo apt-get install swig3.0 -y
-        sudo ln -s /usr/bin/swig3.0 /usr/bin/swig
-      displayName: Install apt packages
-    - task: Cache@2
-      inputs:
-        key: 'python | "$(Agent.OS)" | dependencies/*.txt'
-        restoreKeys: | 
-          python | "$(Agent.OS)"
-          python
-        path: $(PIP_CACHE_DIR)
-      displayName: Cache pip packages
+        # pylint requires newer typing extension. Override requirements in tensorflow
+        python -m pip install "typing-extensions>=3.10"
+      displayName: Resolve dependency version
+
     - script: |
-        set -e
-        python -m pip install -U -r dependencies/setup.txt
-        python -m pip install -r dependencies/develop.txt
-        python -m pip install -r dependencies/required.txt
-        python -m pip install -r dependencies/recommended.txt
-        python -m pip install -r dependencies/required_extra.txt
-        python -m pip install "typing-extensions>=3.10"  # pylint requires newer typing extension. Override requirements in tensorflow
-      displayName: Install requirements
-    - script: python -m pylint --rcfile pylintrc nni
+        python -m pylint --rcfile pylintrc nni
       displayName: pylint
+
     - script: |
         set -e
         python -m flake8 nni --count --select=E9,F63,F72,F82 --show-source --statistics
@@ -92,103 +54,45 @@ stages:
   - job: typescript
     pool:
       vmImage: ubuntu-latest
-    variables:
-      YARN_CACHE_FOLDER: $(Pipeline.Workspace)/.yarn
     steps:
-    - task: NodeTool@0
-      inputs:
-        versionSpec: 16.3.0
-      displayName: Configure Node.js version
-    - task: Cache@2
-      inputs:
-        key: 'yarn | "$(Agent.OS)" | ts/**/yarn.lock, !**/node_modules/**'
-        restoreKeys: |
-          yarn | "$(Agent.OS)"
-        path: $(YARN_CACHE_FOLDER)
-      displayName: Cache yarn packages
+    - template: templates/install-dependencies.yml
+      parameters:
+        platform: ubuntu-latest
+
     - script: |
         set -e
-        cd ts/nni_manager
-        yarn
-        yarn eslint
+        yarn --cwd ts/nni_manager
+        yarn --cwd ts/nni_manager eslint
       displayName: ESLint (NNI Manager)
+
     - script: |
         set -e
-        cd ts/webui
-        yarn
-        yarn eslint
+        yarn --cwd ts/webui
+        yarn --cwd ts/webui eslint
       displayName: ESLint (WebUI)
 
 
+  # To reduce debug cost, steps are sorted differently on each platform,
+  # so that a bug in any module will cause at least one platform to fail quickly.
 - stage: test
   jobs:
   - job: ubuntu_latest
     pool:
       vmImage: ubuntu-latest
-    variables:
-      PIP_CACHE_DIR: $(Pipeline.Workspace)/.pip
-      YARN_CACHE_FOLDER: $(Pipeline.Workspace)/.yarn
 
     steps:
-    - task: UsePythonVersion@0
-      inputs:
-        versionSpec: 3.9
-      displayName: Configure Python version
-
-    - task: NodeTool@0
-      inputs:
-        versionSpec: 16.3.0
-      displayName: Configure Node.js version
-
-    - script: |
-        sudo apt-get install -y pandoc
-        sudo apt-get remove swig -y
-        sudo apt-get install swig3.0 -y
-        sudo ln -s /usr/bin/swig3.0 /usr/bin/swig
-      displayName: Install apt packages
-
-    - task: Cache@2
-      inputs:
-        key: 'python | "$(Agent.OS)" | latest | dependencies/*.txt'
-        restoreKeys: | 
-          python | "$(Agent.OS)"
-          python
-        path: $(PIP_CACHE_DIR)
-      displayName: Cache pip packages
-
-    - task: Cache@2
-      inputs:
-        key: 'yarn | "$(Agent.OS)" | latest | ts/**/yarn.lock, !**/node_modules/**'
-        restoreKeys: |
-          yarn | "$(Agent.OS)"
-        path: $(YARN_CACHE_FOLDER)
-      displayName: Cache yarn packages
-
-    - script: |
-        set -e
-        python -m pip install -U -r dependencies/setup.txt
-        python -m pip install -r dependencies/develop.txt
-        echo "##vso[task.setvariable variable=PATH]${HOME}/.local/bin:${PATH}"
-      displayName: Install Python tools
+    - template: templates/install-dependencies.yml
+      parameters:
+        platform: ubuntu-latest
 
     - script: |
         python setup.py develop
-        mkdir -p coverage
+        echo "##vso[task.setvariable variable=PATH]${HOME}/.local/bin:${PATH}"
       displayName: Install NNI
 
     - script: |
         set -e
-        python -m pip install -r dependencies/recommended.txt
-        python -m pip install -e .[PPOTuner,DNGO]
-      displayName: Install extra dependencies
-
-    # Need del later
-    - script: |
-        python test/vso_tools/interim_patch.py
-      displayName: Torch utils tensorboard interim patch
-
-    - script: |
-        set -e
+        mkdir -p coverage
         cd test
         python -m pytest ut --cov-config=.coveragerc \
           --ignore=ut/compression/v1/test_pruners.py \
@@ -203,10 +107,8 @@ stages:
       displayName: Python unit test
 
     - script: |
-        set -e
-        cd ts/nni_manager
-        yarn test
-        cp coverage/cobertura-coverage.xml ../../coverage/typescript.xml
+        yarn --cwd ts/nni_manager test
+        cp ts/nni_manager/coverage/cobertura-coverage.xml coverage/typescript.xml
       displayName: TypeScript unit test
 
     - task: PublishTestResults@2
@@ -230,71 +132,17 @@ stages:
   - job: ubuntu_legacy
     pool:
       vmImage: ubuntu-18.04
-    variables:
-      PIP_CACHE_DIR: $(Pipeline.Workspace)/.pip
-      YARN_CACHE_FOLDER: $(Pipeline.Workspace)/.yarn
 
     steps:
-    - task: UsePythonVersion@0
-      inputs:
-        versionSpec: 3.7
-      displayName: Configure Python version
-
-    - task: NodeTool@0
-      inputs:
-        versionSpec: 16.3.0
-      displayName: Configure Node.js version
-
-    - script: |
-        sudo apt-get install -y pandoc
-        sudo apt-get remove swig -y
-        sudo apt-get install swig3.0 -y
-        sudo ln -s /usr/bin/swig3.0 /usr/bin/swig
-      displayName: Install apt packages
-
-    - task: Cache@2
-      inputs:
-        key: 'python | "$(Agent.OS)" | legacy | dependencies/*.txt'
-        restoreKeys: | 
-          python | "$(Agent.OS)"
-          python
-        path: $(PIP_CACHE_DIR)
-      displayName: Cache pip packages
-
-    - task: Cache@2
-      inputs:
-        key: 'yarn | "$(Agent.OS)" | legacy | ts/**/yarn.lock, !**/node_modules/**'
-        restoreKeys: |
-          yarn | "$(Agent.OS)"
-        path: $(YARN_CACHE_FOLDER)
-      displayName: Cache yarn packages
-
-    - script: |
-        set -e
-        cd dependencies
-        python -m pip install -U -r setup.txt
-        # FIXME: Version resolving is a complete mess. Now it works magically.
-        python -m pip install numpy==1.21.5
-        python -m pip install -r develop.txt
-        python -m pip install -r required.txt
-        python -m pip install -r required_extra.txt
-        python -m pip install -r recommended_legacy.txt
-        python -m pip install numpy==1.21.5
-        python -m pip install pytorch-lightning==1.5.9
-        echo "##vso[task.setvariable variable=PATH]${HOME}/.local/bin:${PATH}"
-      displayName: Install Python dependencies
+    - template: templates/install-dependencies.yml
+      parameters:
+        platform: ubuntu-legacy
 
     - script: |
         python setup.py develop
+        echo "##vso[task.setvariable variable=PATH]${HOME}/.local/bin:${PATH}"
       displayName: Install NNI
 
-    # Need del later
-    - script: |
-        set -e
-        python test/vso_tools/interim_patch.py
-      displayName: Torch utils tensorboard interim patch
-
-
     - script: |
         cd test
         python nni_test/nnitest/run_tests.py --config config/pr_tests.yml
@@ -308,157 +156,29 @@ stages:
       displayName: Python unit test
 
     - script: |
-        set -e
-        cd ts/nni_manager
-        yarn test
+        yarn --cwd ts/nni_manager test
       displayName: TypeScript unit test
 
-  - job: macos
-    pool:
-      vmImage: macOS-latest
-    variables:
-      PIP_CACHE_DIR: $(Pipeline.Workspace)/.pip
-      YARN_CACHE_FOLDER: $(Pipeline.Workspace)/.yarn
-    timeoutInMinutes: 90  # macos test need extra time
-
-    steps:
-    - task: UsePythonVersion@0
-      inputs:
-        versionSpec: 3.9
-      displayName: Configure Python version
-
-    - task: NodeTool@0
-      inputs:
-        versionSpec: 16.3.0
-      displayName: Configure Node.js version
-
-    - script: |
-        brew install swig@3
-        rm -f /usr/local/bin/swig
-        ln -s /usr/local/opt/swig\@3/bin/swig /usr/local/bin/swig
-      displayName: Install brew packages
-
-    - task: Cache@2
-      inputs:
-        key: 'python | "$(Agent.OS)" | dependencies/*.txt'
-        restoreKeys: | 
-          python | "$(Agent.OS)"
-          python
-        path: $(PIP_CACHE_DIR)
-      displayName: Cache pip packages
-
-    - task: Cache@2
-      inputs:
-        key: 'yarn | "$(Agent.OS)" | ts/**/yarn.lock, !**/node_modules/**'
-        restoreKeys: |
-          yarn | "$(Agent.OS)"
-        path: $(YARN_CACHE_FOLDER)
-      displayName: Cache yarn packages
-
-    - script: |
-        set -e
-        python -m pip install -U -r dependencies/setup.txt
-        python -m pip install -r dependencies/develop.txt
-        echo "##vso[task.setvariable variable=PATH]${PATH}:${HOME}/.local/bin"
-      displayName: Install Python tools
-
-    - script: |
-        python setup.py develop
-      displayName: Install NNI
-
-    - script: |
-        set -e
-        export CI=true
-        (cd ts/nni_manager && yarn test --exclude test/core/nnimanager.test.ts)
-      displayName: TypeScript unit test
-
-    - script: |
-        set -e
-        python -m pip install -r dependencies/recommended.txt
-        python -m pip install -e .[SMAC,BOHB,PPOTuner,DNGO]
-      displayName: Install extra dependencies
-
-    # Need del later
-    - script: |
-        set -e
-        python test/vso_tools/interim_patch.py
-      displayName: Torch utils tensorboard interim patch
-
-    - script: |
-        cd test
-        python -m pytest ut
-      displayName: Python unit test
-
-    - script: |
-        cd test
-        python nni_test/nnitest/run_tests.py --config config/pr_tests.yml
-      displayName: Simple integration test
-
   - job: windows
     pool:
       vmImage: windows-latest
-    variables:
-      PIP_CACHE_DIR: $(Pipeline.Workspace)/.pip
-      YARN_CACHE_FOLDER: $(Pipeline.Workspace)/.yarn
-    timeoutInMinutes: 120  # windows test need extra time
 
     steps:
-    - task: UsePythonVersion@0
-      inputs:
-        versionSpec: 3.9
-      displayName: Configure Python version
-
-    - task: NodeTool@0
-      inputs:
-        versionSpec: 16.3.0
-      displayName: Configure Node.js version
-
-    - task: Cache@2
-      inputs:
-        key: 'python | "$(Agent.OS)" | dependencies/*.txt'
-        restoreKeys: | 
-          python | "$(Agent.OS)"
-          python
-        path: $(PIP_CACHE_DIR)
-      displayName: Cache pip packages
-
-    - task: Cache@2
-      inputs:
-        key: 'yarn | "$(Agent.OS)" | ts/**/yarn.lock, !**/node_modules/**'
-        restoreKeys: |
-          yarn | "$(Agent.OS)"
-        path: $(YARN_CACHE_FOLDER)
-      displayName: Cache yarn packages
-
-    - script: |
-        set -e
-        python -m pip install -U -r dependencies/setup.txt
-        python -m pip install -r dependencies/develop.txt
-      displayName: Install Python tools
+    - template: templates/install-dependencies.yml
+      parameters:
+        platform: windows
 
     - script: |
         python setup.py develop --no-user
       displayName: Install NNI
 
-    - script: |
-        python -m pip install -r dependencies/recommended.txt
-        python -m pip install -e .[DNGO]
-      displayName: Install extra dependencies
-
-    # Need del later
-    - script: |
-        set -e
-        python test/vso_tools/interim_patch.py
-      displayName: Torch utils tensorboard interim patch
-
     - script: |
         cd test
         python -m pytest ut
       displayName: Python unit test
 
     - script: |
-        cd ts/nni_manager
-        yarn test
+        yarn --cwd ts/nni_manager test
       displayName: TypeScript unit test
 
     - script: |
@@ -466,7 +186,30 @@ stages:
         python nni_test/nnitest/run_tests.py --config config/pr_tests.yml
       displayName: Simple integration test
 
+  - job: macos
+    pool:
+      vmImage: macOS-latest
 
-trigger:
-  branches:
-    exclude: [ l10n_master ]
+    steps:
+    - template: templates/install-dependencies.yml
+      parameters:
+        platform: macos
+
+    - script: |
+        python setup.py develop
+        echo "##vso[task.setvariable variable=PATH]${HOME}/.local/bin:${PATH}"
+      displayName: Install NNI
+
+    - script: |
+        CI=true yarn --cwd ts/nni_manager test --exclude test/core/nnimanager.test.ts
+      displayName: TypeScript unit test
+
+    - script: |
+        cd test
+        python -m pytest ut
+      displayName: Python unit test
+
+    - script: |
+        cd test
+        python nni_test/nnitest/run_tests.py --config config/pr_tests.yml
+      displayName: Simple integration test

pipelines/templates/cache-dependencies-template.yml

@@ -0,0 +1,74 @@
+parameters:
+- name: platform
+  type: string
+
+steps:
+- template: config-version.yml
+  parameters:
+    platform: ${{ parameters.platform }}
+
+- script: |
+    echo "##vso[task.setvariable variable=PLATFORM]${{ parameters.platform }}"
+    python test/vso_tools/pip_use_prefix.py python-packages
+  displayName: Prepare
+
+- script: |
+    cp dependencies/recommended_legacy.txt dependencies/recommended.txt
+  displayName: (legacy) Activate legacy dependencies
+  condition: and(succeeded(), contains('${{ parameters.platform }}', 'legacy'))
+
+- script: |
+    set -e
+    echo '===== develop ====='
+    ./pip-install -r dependencies/develop.txt
+    echo '===== required ====='
+    ./pip-install -r dependencies/required.txt
+    echo '===== required extra ====='
+    ./pip-install -r dependencies/required_extra.txt
+    echo '===== recommended ====='
+    ./pip-install -r dependencies/recommended.txt
+    echo '===== fix shebang ====='
+    python test/vso_tools/fix_shebang.py python-packages/bin
+  displayName: (POSIX) Install Python packages
+  condition: and(succeeded(), not(contains('${{ parameters.platform }}', 'windows')))
+
+# FIXME: It does not stop on failure. Tried "ErrorActionPreference" with no luck.
+- powershell: |
+    echo '===== develop ====='
+    ./pip-install -r dependencies/develop.txt
+    echo '===== required ====='
+    ./pip-install -r dependencies/required.txt
+    echo '===== required extra ====='
+    ./pip-install -r dependencies/required_extra.txt
+    echo '===== recommended ====='
+    ./pip-install -r dependencies/recommended.txt
+  displayName: (Windows) Install Python packages
+  condition: and(succeeded(), contains('${{ parameters.platform }}', 'windows'))
+
+# FIXME:
+# numpy version may change several times during installation, and it has an ABI change in recent versions.
+# We must make sure ConfigSpaceNNI is built against correct numpy version, or it will fail on import.
+- script: |
+    python -m pip uninstall -y ConfigSpaceNNI
+    ./pip-install --no-cache-dir ConfigSpaceNNI
+  displayName: (Ubuntu) Rebuild ConfigSpaceNNI
+  condition: and(succeeded(), contains('${{ parameters.platform }}', 'ubuntu'))
+
+- script: |
+    yarn --cwd ts/nni_manager install
+  displayName: Install NNI Manager dependencies
+
+- script: |
+    yarn --cwd ts/webui install
+  displayName: Install Web UI dependencies
+
+- script: |
+    python test/vso_tools/pack_dependencies.py $(Build.ArtifactStagingDirectory)
+  displayName: Create cache archive
+
+- task: UniversalPackages@0
+  inputs:
+    command: publish
+    vstsFeedPublish: NNIOpenSource/dependencies
+    vstsFeedPackagePublish: dependencies-${{ parameters.platform }}
+  displayName: Upload cache archive

pipelines/templates/config-version.yml

@@ -0,0 +1,25 @@
+parameters:
+- name: platform
+  type: string
+
+steps:
+- task: UsePythonVersion@0
+  inputs:
+    versionSpec: 3.9
+  displayName: (latest) Configure Python version
+  condition: and(succeeded(), not(contains('${{ parameters.platform }}', 'legacy')))
+
+- task: UsePythonVersion@0
+  inputs:
+    versionSpec: 3.7
+  displayName: (legacy) Configure Python version
+  condition: and(succeeded(), contains('${{ parameters.platform }}', 'legacy'))
+
+- task: NodeTool@0
+  inputs:
+    versionSpec: 16.3.0
+  displayName: Configure Node.js version
+
+- script: |
+    python -m pip install pip==21.3.1 setuptools==60.5.0 wheel==0.37.1
+  displayName: Install PyPA tools

pipelines/templates/install-dependencies.yml

@@ -0,0 +1,56 @@
+parameters:
+- name: platform
+  type: string
+
+steps:
+- template: config-version.yml
+  parameters:
+    platform: ${{ parameters.platform }}
+
+- script: |
+    set -e
+    sudo apt-get install -y swig3.0
+    sudo rm /usr/bin/swig
+    sudo ln -s swig3.0 /usr/bin/swig
+  displayName: (Ubuntu) Downgrade swig
+  condition: and(succeeded(), contains('${{ parameters.platform }}', 'ubuntu'))
+
+- script: |
+    set -e
+    brew install 'swig@3'
+    rm /usr/local/bin/swig
+    ln -s '/usr/local/opt/swig@3/bin/swig' /usr/local/bin/swig
+  displayName: (macOS) Downgrade swig
+  condition: and(succeeded(), contains('${{ parameters.platform }}', 'macos'))
+
+- task: UniversalPackages@0
+  inputs:
+    vstsFeed: NNIOpenSource/dependencies
+    vstsFeedPackage: dependencies-${{ parameters.platform }}
+    vstsPackageVersion: "*"
+  displayName: Download cache
+
+- script: |
+    python test/vso_tools/unpack_dependencies.py
+  displayName: Unpack cache
+
+- script: |
+    mv dependencies/recommended_legacy.txt dependencies/recommended.txt
+  displayName: (legacy) Activate legacy dependencies
+  condition: and(succeeded(), contains('${{ parameters.platform }}', 'legacy'))
+
+- script: |
+    echo '===== develop ====='
+    python -m pip install -r dependencies/develop.txt
+    echo '===== required ====='
+    python -m pip install -r dependencies/required.txt
+    echo '===== required extra ====='
+    python -m pip install -r dependencies/required_extra.txt
+    echo '===== recommended ====='
+    python -m pip install -r dependencies/recommended.txt
+  displayName: Install Python dependencies
+
+# TODO: Delete this after upgrading to PyTorch 1.11.
+- script: |
+    python test/vso_tools/interim_patch.py
+  displayName: Torch utils tensorboard interim patch

test/vso_tools/fix_shebang.py

@@ -0,0 +1,19 @@
+"""
+Change "#!" line to "#!/usr/bin/env python" for all files in directory sys.argv[1].
+"""
+
+from pathlib import Path
+import sys
+
+if sys.platform == 'win32':
+    exit()
+
+for file in Path(sys.argv[1]).iterdir():
+    try:
+        text = file.read_text()
+        assert text.startswith('#!'), 'no shebang'
+        shebang, content = text.split('\n', 1)
+        assert 'python' in shebang, 'not python script'
+        file.write_text('#!/usr/bin/env python\n' + content)
+    except Exception as e:
+        print(f'Skip {file}: {repr(e)}')

test/vso_tools/pack_dependencies.py

@@ -0,0 +1,43 @@
+"""
+Create an archive in sys.argv[1], containing python-packages and node_modules.
+Use unpack_dependencies.py to extract the archive.
+"""
+
+import json
+import os
+from pathlib import Path
+import shutil
+import site
+import sys
+from zipfile import ZIP_DEFLATED, ZipFile
+
+def main() -> None:
+    cache = Path('cache')
+    cache.mkdir()
+    shutil.move('python-packages', 'cache/python-dependencies')
+    shutil.move('ts/nni_manager/node_modules', 'cache/nni-manager-dependencies')
+    shutil.move('ts/webui/node_modules', 'cache/webui-dependencies')
+
+    archive = ZipFile('cache.zip', 'w', ZIP_DEFLATED, compresslevel=9)
+    symlinks = {}
+    empty_dirs = set()
+    for file in sorted(cache.rglob('*')):
+        if file.parent.parent == cache or file.parent.name == 'site-packages':
+            print('Compress', file, flush=True)
+        if file.is_symlink():
+            symlinks[str(file)] = os.readlink(file)  # file.readlink() was added in Python 3.9
+            continue
+        if file.is_dir():
+            empty_dirs.add(str(file))
+        else:
+            archive.write(file)
+        empty_dirs.discard(str(file.parent))
+    archive.writestr('symlinks.json', json.dumps(symlinks, indent=4))
+    archive.writestr('directories.json', json.dumps(list(empty_dirs), indent=4))
+    archive.close()
+
+    assert Path(sys.argv[1]).is_dir()
+    shutil.move('cache.zip', sys.argv[1])
+
+if __name__ == '__main__':
+    main()

test/vso_tools/pip_use_prefix.py

@@ -0,0 +1,38 @@
+"""
+ 0. Create a directory whose name is sys.argv[1].
+ 1. Create a "pip-install" script that uses this directory as prefix.
+ 2. Set PYTHONPATH to use this prefix, via VSO variable.
+"""
+
+import os
+from pathlib import Path
+import sys
+import typing
+
+prefix = Path(sys.argv[1])
+prefix.mkdir(parents=True, exist_ok=True)
+prefix = prefix.resolve()
+
+if sys.platform == 'win32':
+    # Do not use CMD script. It will magically stop pipeline step.
+    script = f'python -m pip install --prefix {prefix} --no-compile --prefer-binary $args'
+    Path('pip-install.ps1').write_text(script + '\n')
+else:
+    script = f'python -m pip install --prefix {prefix} --no-compile --prefer-binary "$@"'
+    Path('pip-install').write_text('#!/bin/bash\n' + script + '\n')
+    os.chmod('pip-install', 0o775)
+
+if sys.platform == 'win32':
+    site_path = prefix / 'Lib/site-packages'
+else:
+    version = f'{sys.version_info.major}.{sys.version_info.minor}'
+    site_path = prefix / f'lib/python{version}/site-packages'
+paths = [
+    Path(typing.__file__).parent,  # With PYTHONPATH, the backport version of typing will mask stdlib version.
+    site_path,
+    os.getenv('PYTHONPATH'),
+]
+path = os.pathsep.join(str(p) for p in paths if p)
+
+print('PYTHONPATH:', path)
+print('##vso[task.setvariable variable=PYTHONPATH]' + path)

test/vso_tools/unpack_dependencies.py

@@ -0,0 +1,56 @@
+"""
+Extract an archive created by pack_dependencies.py.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+import shutil
+import site
+import sys
+from zipfile import ZipFile
+
+def main() -> None:
+    if sys.platform == 'win32':
+        # Strangely user site is not enabled on Windows.
+        prefix = Path(sys.prefix)
+    else:
+        prefix = Path(site.getuserbase())
+
+    print('Extract Python packages to', prefix)
+    print('All Python paths:')
+    print('\n'.join(sys.path), flush=True)
+
+    extract_all(ZipFile('cache.zip'))
+    empty_dirs = json.loads(Path('directories.json').read_text())
+    symlinks = json.loads(Path('symlinks.json').read_text())
+    for dir_ in empty_dirs:
+        Path(dir_).mkdir(parents=True, exist_ok=True)
+    for link, target in symlinks.items():
+        Path(link).symlink_to(target)  # hopefully nobody uses symlink on windows
+
+    move_or_merge(Path('cache/python-dependencies'), prefix)
+    shutil.move('cache/nni-manager-dependencies', 'ts/nni_manager/node_modules')
+    shutil.move('cache/webui-dependencies', 'ts/webui/node_modules')
+
+def extract_all(zf: ZipFile) -> None:
+    # fix a bug in ZipFile.extractall()
+    # see https://stackoverflow.com/questions/39296101
+    for info in zf.infolist():
+        path = zf.extract(info)
+        if info.external_attr > 0xffff:
+            os.chmod(path, info.external_attr >> 16)
+
+def move_or_merge(src: Path | str, dst: Path | str) -> None:
+    if not dst.exists():
+        shutil.move(src, dst)
+    elif dst.is_dir():
+        for file in src.iterdir():
+            move_or_merge(file, dst / file.name)
+    else:
+        print('Skip', dst)
+
+if __name__ == '__main__':
+    main()