pywwt/ci/azure-build-and-test.yml

# Main build jobs for CI/CD on Azure Pipelines.

parameters:
- name: builds
  type: object
  default:

  - name: linux_38
    vmImage: ubuntu-22.04
    vars:
      PYTHON_SERIES: "3.8"

  - name: linux_39
    vmImage: ubuntu-22.04
    vars:
      PYTHON_SERIES: "3.9"

  - name: linux_310
    vmImage: ubuntu-22.04
    vars:
      PYTHON_SERIES: "3.10"

  - name: macos_310
    vmImage: macos-12
    vars:
      PYTHON_SERIES: "3.10"

  - name: windows_310
    vmImage: windows-2022
    vars:
      PYTHON_SERIES: "3.10"

jobs:
- ${{ each build in parameters.builds }}:
  - job: ${{ format('build_{0}', build.name) }}
    pool:
      vmImage: ${{ build.vmImage }}
    variables:
      ${{ insert }}: ${{ build.vars }}
    steps:

    - template: azure-job-setup.yml
      parameters:
        setupBuild: true

    - bash: |
        set -euo pipefail
        source activate-conda.sh
        set -x
        \conda create -y -n build setuptools matplotlib pip pyopengl 'pyqt>=5.12' pyqtwebengine python="$PYTHON_SERIES" qtpy scipy shapely
        conda activate build
        pip install $BASH_WORKSPACE/sdist/*.tar.gz        
      displayName: Install from sdist

    # NB: `set -x` interferes with task.setvariable
    - bash: |
        set -euo pipefail
        echo "##vso[task.setvariable variable=DISPLAY;]:99"
        echo "##vso[task.setvariable variable=QTWEBENGINE_CHROMIUM_FLAGS;]--ignore-gpu-blocklist"
        set -x
        sudo apt-get update
        sudo apt-get install -y \
          libxkbcommon-x11-0 \
          libgl1-mesa-dev
        /sbin/start-stop-daemon --start --quiet --pidfile /tmp/custom_xvfb_99.pid \
          --make-pidfile --background --exec /usr/bin/Xvfb \
          -- :99 -screen 0 1920x1200x24 -ac \
          +extension GLX +render -noreset
        source activate-conda.sh
        conda activate build
        conda install -y xcb-util-keysyms xcb-util-image xcb-util-renderutil xcb-util-wm  # needed to load Qt XCB backend        
      condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
      displayName: Set up Xvfb/OpenGL (Linux)

    # The Azure Pipelines build hosts don't have GPUs. In order to get WebGL
    # rendering properly, we need to enable software OpenGL rendering through
    # QtWebEngine, which is quite a hassle. This is the simplest recipe I can
    # devise:
    - pwsh: |
        $webclient = New-Object System.Net.WebClient
        $url = "https://chocolatey.org/7za.exe"
        Write-Host "Downloading" $url
        $webclient.DownloadFile($url, "7za.exe")
        $url = "https://download.qt.io/development_releases/prebuilt/llvmpipe/windows/opengl32sw-64-mesa_11_2_2-signed.7z"
        $dest = "opengl32sw-64.7z"
        Write-Host "Downloading" $url
        $webclient.DownloadFile($url, $dest)
        .\7za x -bd -y $dest
        $filepath = "C:\Windows\System32\opengl32.dll"
        takeown /F $filepath /A
        icacls $filepath /grant "${env:ComputerName}\${env:UserName}:F"
        Remove-Item -LiteralPath $filepath
        cp opengl32sw.dll $filepath
        echo "##vso[task.setvariable variable=QT_OPENGL;]software"
        echo "##vso[task.setvariable variable=QT_OPENGL_DLL;]opengl32.dll"
        echo "##vso[task.setvariable variable=QTWEBENGINE_CHROMIUM_FLAGS;]--enable-webgl-software-rendering --ignore-gpu-blocklist"        
      condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
      displayName: Set up OpenGL (Windows)

    - bash: |
        echo "##vso[task.setvariable variable=QTWEBENGINE_CHROMIUM_FLAGS;]--ignore-gpu-blocklist"        
      condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin'))
      displayName: Set up OpenGL (macOS)

    - bash: |
        set -euo pipefail
        source activate-conda.sh
        conda activate build
        set -x
        \conda install -y pytest-qt pytest-remotedata pytest-timeout
        python setup.py build
        pytest -v -s pywwt -p no:warnings --timeout=1800 --timeout_method=thread        
      displayName: Test
      env:
        PYWWT_TEST_IMAGE_DIR: $(Build.ArtifactStagingDirectory)

    # We used to publish the test image artifacts regardless of whether the test
    # succeeded or failed (using `condition: true`). This led to an annoying
    # behavior, though, where if an image test failed spuriously, you'd have to
    # rerun the entire build: if the test succeeded the next time, the
    # publication step would fail because you can't upload artifacts under the
    # same name more than once. So now, we only publish the images if the tests
    # failed: that way, "rerun failed jobs" will be able to resolve a spurious
    # failure efficiently. This is especially valuable in Cranko release and RC
    # builds. If an image test fails *non*-spuriously, you'll only be able to
    # obtain artifacts from the first failure in given build ... but that was
    # what was happening with `condition: true` anyway.
    #
    # We might want to turn this into some kind of top-level pipeline parameter
    # because I think there will be occasions where it would be nice to gather
    # the image data for all of the image tests, including the successful ones.
    # But that's more of a once-in-a-long-while thing, while the rerun issue
    # rears in say ~5% of PR builds.

    - task: PublishPipelineArtifact@1
      displayName: Publish failing test images
      condition: not(succeeded())
      inputs:
        targetPath: $(Build.ArtifactStagingDirectory)
        artifactName: ${{ format('test_images_{0}', build.name) }}

    - bash: |
        set -euo pipefail
        source activate-conda.sh
        conda activate build
        set -x
        \conda install -y jupyterlab nbclassic
        jupyter nbclassic-extension list
        jupyter nbclassic-serverextension list
        jupyter labextension list        
      displayName: Print Jupyter extension status

    - bash: |
        set -euo pipefail
        source activate-conda.sh
        conda activate build
        set -x
        python .check_enabled.py        
      displayName: Check Jupyter extension status

- job: coverage
  pool:
    vmImage: ubuntu-latest
  steps:
  - template: azure-job-setup.yml
    parameters:
      setupBuild: true

  - bash: |
      set -euo pipefail
      source activate-conda.sh
      set -x
      \conda create -y -n build setuptools matplotlib pip pyopengl 'pyqt>=5.12' pyqtwebengine python=3.9 qtpy scipy shapely
      conda activate build
      pip install $BASH_WORKSPACE/sdist/*.tar.gz      
    displayName: Install from sdist

  # NB: `set -x` interferes with task.setvariable
  - bash: |
      set -euo pipefail
      echo "##vso[task.setvariable variable=DISPLAY;]:99"
      echo "##vso[task.setvariable variable=QTWEBENGINE_CHROMIUM_FLAGS;]--ignore-gpu-blocklist"
      set -x
      sudo apt-get update
      sudo apt-get install -y \
        libxkbcommon-x11-0 \
        libgl1-mesa-dev
      /sbin/start-stop-daemon --start --quiet --pidfile /tmp/custom_xvfb_99.pid \
        --make-pidfile --background --exec /usr/bin/Xvfb \
        -- :99 -screen 0 1920x1200x24 -ac \
        +extension GLX +render -noreset      
    condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
    displayName: Set up Xvfb/OpenGL (Linux)

  - bash: |
      set -euo pipefail
      source activate-conda.sh
      conda activate build
      set -x
      \conda install -y pytest-cov pytest-remotedata pytest-timeout
      python setup.py build
      pytest -v -s pywwt -p no:warnings --timeout=1800 --timeout_method=thread --cov-report=xml --cov=pywwt      
    displayName: Test with coverage

  - bash: bash <(curl -s https://codecov.io/bash)
    displayName: Upload coverage results

- job: docs
  pool:
    vmImage: ubuntu-latest
  steps:
  - template: azure-job-setup.yml
    parameters:
      setupBuild: true

  # We install some extra deps here to make the docs build happy (it needs to
  # import everything) while avoiding clashes between Conda and pip.
  - bash: |
      set -euo pipefail
      source activate-conda.sh
      set -x
      \conda create -y -n build \
        astropy \
        astropy-sphinx-theme \
        ipykernel \
        jupyter_sphinx \
        matplotlib \
        nbclassic \
        numpydoc \
        pip \
        pyopengl \
        'pyqt>=5.12' \
        pyqtwebengine \
        python=3.9 \
        qtpy \
        scipy \
        setuptools \
        shapely \
        sphinx \
        sphinx-automodapi
      conda activate build
      pip install $BASH_WORKSPACE/sdist/*.tar.gz      
    displayName: Install from sdist

  # 2021 June: we used to finish with a `make linkcheck`, but Matplotlib's
  # intersphinx file currently causes that step to fail in a way that we can't
  # do anything about. We should put it back once that's straightened out.
  - bash: |
      set -euo pipefail
      source activate-conda.sh
      conda activate build
      set -x
      cd docs
      make html      
    displayName: Build docs

- job: codestyle
  pool:
    vmImage: ubuntu-latest
  steps:
  - template: azure-job-setup.yml
    parameters:
      setupBuild: true

  - bash: |
      set -euo pipefail
      source activate-conda.sh
      set -x
      \conda create -y -n build setuptools pip python=3.9
      conda activate build
      pip install $BASH_WORKSPACE/sdist/*.tar.gz      
    displayName: Install from sdist

  - bash: |
      set -euo pipefail
      source activate-conda.sh
      conda activate build
      set -x
      \conda install -y flake8
      flake8 --max-line-length=200 pywwt      
    displayName: Check flake8