Extend testing CI to re-run conda tests in self-contained environment (#597)

* run conda tests in linux container

* use steps-conda template

* use windows container

* use mcr for container

* use powershell container

* use windowsservercore tag for powershell container

* rollback to nanoserver version 1809

* use window server core instead of nano

* use nanoserver with powershell flag set to false

* add empty files for acr management

* Update Dockerfile to download conda for powershell

* use acr image for windows conda tests

* run container in docker task

* use container with qdkimages-acr endpoint

* use conda directly in scripts

* update syntax to create, install and activate conda environment

* use conda to activate environment

* call conda init

* separate conda commands into individual calls

* activate conda env in steps-conda

* call conda directly

* remove conda for pwsh

* Include eval conda shell.bash hook

* Use script task and pwsh inline

* Make call to source activate conda_env

* break up steps for conda environment creation

* call conda hook before activation

* call conda activate on all images

* split conda env steps into pwsh and script tasks

* install conda-build in base env instead of conda_env

* run pwsh from pwsh

* use variables for more concise syntax

* add dockerfile with miniconda installed

* consolidate arguments to tasks

* consolidate tasks to use variables

* install powershell to linux container

* install git to linux image

* add readme for dockerfiles

* Combine steps to single layer to reduce image size

Co-authored-by: Cassandra Granade <chgranad@microsoft.com>

* avoid adding conda to path to prevent dependency leakage

* refactor dockerfile

* update conda path for linux

* add conda path for mac

* activate conda shell hook for conda creation and package install

* update dockerfile and add new action workflow

* update images to be from public

* attempt to use windows-latest image and container

* set e2e ref back to main

* Push only when Dockerfile images are changed

Co-authored-by: Cassandra Granade <chgranad@microsoft.com>

* Remove unused variable

Co-authored-by: Cassandra Granade <chgranad@microsoft.com>

* Refactor image pushing to reusable function

Co-authored-by: Cassandra Granade <chgranad@microsoft.com>

* use containers from mcr instead of internal acr

Co-authored-by: Cassandra Granade <chgranad@microsoft.com>

.github/workflows/push-selfcontained.yml

@@ -0,0 +1,94 @@
+name: Push self-contained images on Dockerfile change
+
+on:
+  push:
+    branches:
+    - main
+    paths:
+    - 'images/test-environments/**/Dockerfile`
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: Azure Login
+      uses: Azure/login@v1
+      with:
+        creds: ${{ secrets.AZURE_CREDENTIALS }}
+    - name: Docker Login
+      run: az acr login -n qdkimages
+    - name: Build and push Docker image
+      run: |
+        $ImageTag = "${{ github.sha }}"
+
+        function Update-Image() {
+            param(
+                [string] $Path
+                [string] $RepoName,
+                [string] $LocalTag,
+            );
+            $fullRepoName = "public/quantum/$RepoName";
+            $remoteRepo = "${{ secrets.ACR_REGISTRY }}/${RepoName}";
+
+            docker build $Path --tag $LocalTag;
+
+            docker tag $LocalTag "${remoteRepo}:${LocalTag}";
+            docker push "${remoteRepo}:${LocalTag}";
+
+            docker tag $LocalTag "${remoteRepo}:latest";
+            docker push "${remoteRepo}:latest";
+        }
+        Update-Image `
+            -RepoName "linux-selfcontained" `
+            -LocalTag "${ImageTag}-linux" `
+            -Path images/test-environments/linux;
+            
+        Update-Image `
+            -RepoName "windows-selfcontained" `
+            -LocalTag "${ImageTag}-windows" `
+            -Path images/test-environments/windows;
+      shell: pwsh
+    - name: Wait for images to publish
+      run: |
+        function Test-Manifest {
+          param(
+            [string]$OS
+          );
+
+          try {
+            $manifest = Invoke-RestMethod `
+              "https://mcr.microsoft.com/v2/quantum/samples/manifests/${ImageTag}-$OS" `
+              -ErrorAction Continue;
+            Write-Verbose $manifest;
+            return $true;
+          } catch {
+            return $false;
+          }
+        }
+
+        $LinuxImageAvailable = $false;
+        $WindowsImageAvailable = $false;
+
+        $CheckInterval = 30; # [seconds]
+        while (-not $LinuxImageAvailable -and -not $WindowsImageAvailable) {
+          if (Test-Manifest -OS "linux" -and Test-Manifest -OS "windows") {
+            Write-Host "##[info] Linux and Windows images $ImageTag now available on mcr.microsoft.com, proceeding.";
+  
+            $LinuxImageAvailable = $true;
+            $WindowsImageAvailable = $true;
+          } else if (Test-Manifest -OS "linux") {
+            Write-Host "##[info] Linux image $ImageTag now available on mcr.microsoft.com, proceeding."
+            Write-Host "##[info] Windows image $ImageTag not yet available on mcr.microsoft.com, waiting $CheckInterval seconds.";
+            $LinuxImageAvailable = $true;
+          } else if (Test-Manifest -OS "windows") {
+            Write-Host "##[info] Windows image $ImageTag now available on mcr.microsoft.com, proceeding.";
+            Write-Host "##[info] Linux image $ImageTag not yet available on mcr.microsoft.com, waiting $CheckInterval seconds.";
+            $WindowsImageAvailable = $true;
+          } else {
+            Write-Host "##[info] Neither Linux nor Windows images are available on mcr.microsoft.com, waiting $CheckInterval seconds.";
+          }
+
+          Start-Sleep -Seconds $CheckInterval;
+        }

build/ci.yml

@@ -21,6 +21,7 @@ variables:
   Build.Minor: 18
   Drops.Dir: $(Build.ArtifactStagingDirectory)/drops
   IQSharp.Hosting.Env: 'build-agent-iqsharp'
+  agent.preferPowerShellOnContainers: false
 
 jobs:
 - job: "iqsharp"
@@ -48,26 +49,38 @@ jobs:
   - template: steps-selfcontained.yml
   condition: ne(variables['Enable.Conda'], 'false')
 
-- job: "pack_conda_linux"
-  dependsOn: "pack_selfcontained"
-  pool:
-    vmImage: 'ubuntu-latest'
-  steps:
-  - template: steps-conda.yml
-  condition: ne(variables['Enable.Conda'], 'false')
-
 - job: "pack_conda_macos"
   dependsOn: "pack_selfcontained"
   pool:
     vmImage: 'macOS-latest'
+  variables:
+    OS: 'mac'
+    CondaShellHook: "$('/usr/local/bin/conda' 'shell.bash' 'hook')"
   steps:
   - template: steps-conda.yml
   condition: ne(variables['Enable.Conda'], 'false')
 
-- job: "pack_conda_win"
+- job: "pack_conda_linux"
   dependsOn: "pack_selfcontained"
   pool:
-    vmImage: 'windows-2019'
+    vmImage: 'ubuntu-latest'
+  container: mcr.microsoft.com/quantum/linux-selfcontained:latest
+  variables:
+    OS: 'linux'
+    CondaPath: '/miniconda/bin'
+    CondaShellHook: "$('/miniconda/bin/conda' 'shell.bash' 'hook')"
+  steps:
+  - template: steps-conda.yml
+  condition: ne(variables['Enable.Conda'], 'false')
+
+- job: "pack_conda_windows"
+  dependsOn: "pack_selfcontained"
+  pool:
+    vmImage: 'windows-latest'
+  container: mcr.microsoft.com/quantum/windows-selfcontained:latest
+  variables:
+    OS: 'windows'
+    CondaPath: 'C:\Miniconda3\Scripts'
   steps:
   - template: steps-conda.yml
   condition: ne(variables['Enable.Conda'], 'false')

build/steps-conda.yml

@@ -1,9 +1,8 @@
 ##
-# Build and test IQ#.
+# Move build artifacts and create conda environment
 ##
 
 steps:
-
 ##
 # Pre-reqs
 ##
@@ -12,29 +11,87 @@ steps:
     artifactName: iqsharp
     downloadPath: '$(System.DefaultWorkingDirectory)/artifacts'
 
-- pwsh: |
-    Move-Item "$Env:SYSTEM_DEFAULTWORKINGDIRECTORY/artifacts/iqsharp/drops" "$Env:SYSTEM_DEFAULTWORKINGDIRECTORY/drops"
+- pwsh: Move-Item "$Env:SYSTEM_DEFAULTWORKINGDIRECTORY/artifacts/iqsharp/drops" "$Env:SYSTEM_DEFAULTWORKINGDIRECTORY/drops"
   displayName: "Move build artifacts to IQ# working directory"
 
-- task: CondaEnvironment@1
-  inputs:
-    packageSpecs: "python=3.7 pip setuptools pytest jupyter numpy conda-build=3.18.8 conda-package-handling=1.3.11 brotlipy"
-  displayName: 'Use conda environment w/ Python 3.7'
+##
+# Create conda environment
+##
+- script: |
+    eval "$(CondaShellHook)"
+    conda create --yes --quiet --name conda_env
+  displayName: Create Anaconda environment
+  condition: or(eq(variables.OS, 'linux'), eq(variables.OS, 'mac'))
+
+- pwsh: |
+    $(CondaPath)/conda.exe create --yes --quiet --name conda_env
+  displayName: Create Anaconda environment pwsh
+  condition: eq(variables.OS, 'windows')
 
 ##
-# Pack conda packages
+# Install conda packages
 ##
-- pwsh: .\pack-conda.ps1
+- script: |
+    eval "$(CondaShellHook)"
+    conda install --yes --quiet --name base conda-build=3.18.8
+    conda install --yes --quiet --name conda_env python=3.7 pip setuptools pytest jupyter numpy conda-build=3.18.8 conda-package-handling=1.3.11
+  displayName: Install Anaconda packages
+  condition: or(eq(variables.OS, 'linux'), eq(variables.OS, 'mac'))
+
+- pwsh: |
+    $(CondaPath)/conda.exe install --yes --quiet --name base conda-build=3.18.8
+    $(CondaPath)/conda.exe install --yes --quiet --name conda_env python=3.7 pip setuptools pytest jupyter numpy conda-build=3.18.8 conda-package-handling=1.3.11
+  displayName: Install Anaconda packages
+  condition: eq(variables.OS, 'windows')
+
+##
+# Pack conda packages for each OS
+##
+- script: |
+    eval "$(CondaShellHook)"
+    conda activate conda_env
+    conda info
+    pwsh -NoProfile -Command ./pack-conda.ps1
   displayName: "Packing IQ# packages"
   workingDirectory: '$(System.DefaultWorkingDirectory)/build'
+  condition: and(succeeded(), ne(variables['Skip.Tests'], 'true'), or(eq(variables.OS, 'linux'), eq(variables.OS, 'mac')))
+
+# Running pack-conda.ps1 from a new, independent instance of pwsh
+# prevents Azure Pipelines from interpreting the stderr stream from
+# conda-build as an exception stream, making it easier to capture
+# logging information without causing spurious failures.
+#
+# Note that any actual failures in conda-build can be detected by examining
+# exit codes and by testing the artifacts produced by pack-conda.ps1.
+- pwsh: |
+    (& "$(CondaPath)/conda.exe" "shell.powershell" "hook") | Out-String | Invoke-Expression
+    conda activate conda_env
+    conda info
+    pwsh -NoProfile -Command ./pack-conda.ps1
+  displayName: "Packing IQ# packages"
+  workingDirectory: '$(System.DefaultWorkingDirectory)/build'
+  condition: eq(variables.OS, 'windows')
 
 ##
-# Test conda packages
+# Test conda packages for each OS
 ##
-- pwsh: .\test-conda.ps1
+- script: |
+    eval "$(CondaShellHook)"
+    conda activate conda_env
+    conda info
+    pwsh -NoProfile -Command ./test-conda.ps1
   displayName: "Testing IQ# packages"
   workingDirectory: '$(System.DefaultWorkingDirectory)/build'
-  condition: and(succeeded(), ne(variables['Skip.Tests'], 'true'))
+  condition: and(succeeded(), ne(variables['Skip.Tests'], 'true'), or(eq(variables.OS, 'linux'), eq(variables.OS, 'mac')))
+
+- pwsh: |
+    (& "$(CondaPath)/conda.exe" "shell.powershell" "hook") | Out-String | Invoke-Expression
+    conda activate conda_env
+    conda info
+    pwsh -NoProfile -Command ./test-conda.ps1
+  displayName: "Testing IQ# packages"
+  workingDirectory: '$(System.DefaultWorkingDirectory)/build'
+  condition: and(succeeded(), ne(variables['Skip.Tests'], 'true'), eq(variables.OS, 'windows'))
 
 ##
 # Publish build artifacts.
@@ -44,4 +101,4 @@ steps:
   condition: succeededOrFailed()
   inputs:
     PathtoPublish: '$(Build.ArtifactStagingDirectory)'
-    artifactName: iqsharp-conda
+    artifactName: iqsharp-conda

images/test-environments/README.md

@@ -0,0 +1,12 @@
+# Self-contained environments for conda testing and packing
+
+Testing CI runs conda testing and packing on the build agent host. This can make it difficult to find and diagnose issues caused by missing dependencies. For example the build agent host can inadvertently provide dependencies that should be provided by conda packages themselves. Conda testing and packing should be executed from within a brand-new Docker container, in order to emulate a minimal environment as may be installed by users.
+
+Each Dockerfile in this directory represents the contained environments to be used from the build agent for conda testing and packing. Whenever any of these files are changed the new images will be built and pushed to the internal MCR `msint.azure.io`. MacOS is not supported as a pipeline Container job and requires further investigation.
+
+Each Dockerfile includes instructions to download the following tools:
+
+* PowerShell 7
+* Miniconda3
+* Python 3.7
+* Git

images/test-environments/linux/Dockerfile

@@ -0,0 +1,31 @@
+ARG CONDA_VER=latest
+ARG UBUNTU_VER=20.04
+ARG OS_TYPE=x86_64
+ARG PY_VER=3.7
+
+FROM ubuntu:${UBUNTU_VER}
+
+RUN apt-get update && apt-get install -yq curl wget jq
+
+ARG CONDA_VER
+ARG OS_TYPE
+# Install miniconda to /miniconda
+RUN curl -LO "http://repo.continuum.io/miniconda/Miniconda3-${CONDA_VER}-Linux-${OS_TYPE}.sh" && \
+    bash Miniconda3-${CONDA_VER}-Linux-${OS_TYPE}.sh -p /miniconda -b && \
+    rm Miniconda3-${CONDA_VER}-Linux-${OS_TYPE}.sh
+
+ARG UBUNTU_VER
+# Download the Microsoft repository GPG keys
+RUN wget -q "https://packages.microsoft.com/config/ubuntu/${UBUNTU_VER}/packages-microsoft-prod.deb"
+# Register the Microsoft repository GPG keys
+RUN dpkg -i packages-microsoft-prod.deb
+
+ARG DEBIAN_FRONTEND=noninteractive
+# Update the list of packages after we added packages.microsoft.com
+RUN apt-get update && \
+    apt-get -y install \
+        git \
+        powershell && \
+    # We clean the apt cache at the end of each apt command so that the caches
+    # don't get stored in each layer.
+    apt-get clean && rm -rf /var/lib/apt/lists/

images/test-environments/windows/Dockerfile

@@ -0,0 +1,17 @@
+FROM mcr.microsoft.com/windows/servercore:ltsc2022 AS buildbase
+
+RUN powershell (New-Object System.Net.WebClient).DownloadFile('https://repo.anaconda.com/miniconda/Miniconda3-py37_4.11.0-Windows-x86_64.exe', 'Miniconda3.exe') && \
+    powershell (Get-FileHash .\Miniconda3.exe).Hash -eq 'b33797064593ab2229a0135dc69001bea05cb56a20c2f243b1231213642e260a' && \
+    powershell Unblock-File Miniconda3.exe && \
+    Miniconda3.exe /InstallationType=JustMe /RegisterPython=1 /S /D=C:\Miniconda3
+
+FROM mcr.microsoft.com/powershell:lts-windowsservercore-ltsc2022
+
+COPY --from=buildbase C:/Miniconda3 C:/Miniconda3
+
+USER ContainerAdministrator
+
+RUN C:\Miniconda3\Library\bin\conda init && \
+    C:\Miniconda3\Library\bin\conda clean -afy
+
+CMD ["cmd"]