azureml-assets/.github/workflows/assets-release.yaml

name: assets-release

on:
  push:
    branches:
      - main
    paths-ignore:
      - .github/**
      - scripts/**
  schedule:
    - cron: '30 6 * * *'
  workflow_dispatch:
    inputs:
      asset_dirs:
        description: Asset directories
        default: .,!test,!scripts
        required: true

concurrency: ${{ github.workflow }}

env:
  default_asset_dirs: .,!test,!scripts
  main_dir: main
  release_dir: release
  scripts_azureml_assets_dir: scripts/azureml-assets
  scripts_assets_dir: scripts/azureml-assets/azureml/assets
  scripts_environment_dir: scripts/azureml-assets/azureml/assets/environment
  changed_assets_artifact: changed-assets
  releasable_assets_artifact: releasable-assets
  asset_config_filename: asset.yaml

defaults:
  run:
    shell: bash

jobs:
  identify-updated-assets:
    name: Identify updated assets
    runs-on: ubuntu-latest

    steps:
      - name: Clone branch
        uses: actions/checkout@v3
        with:
          fetch-depth: 1
          path: ${{ env.main_dir }}
      
      - name: Clone release branch
        uses: actions/checkout@v3
        with:
          ref: release
          fetch-depth: 0
          path: ${{ env.release_dir }}
      
      - name: Use Python 3.8 or newer
        uses: actions/setup-python@v3
        with:
          python-version: '>=3.8'
    
      - name: Install dependencies
        run: pip install -e $main_dir/$scripts_azureml_assets_dir
      
      # TODO: Consider using Git to detect changes
      - name: Find updated assets
        id: find-updated-assets
        run: python -u $scripts_assets_dir/update_assets.py -i "${{ github.event.inputs.asset_dirs || env.default_asset_dirs }}" -r ${{ github.workspace }}/$release_dir -o ${{ runner.temp }}/$changed_assets_artifact
        working-directory: ${{ env.main_dir }}

      - name: Upload changed assets
        uses: actions/upload-artifact@v3
        if: steps.find-updated-assets.outputs.updated_count > 0
        with:
          name: ${{ env.changed_assets_artifact }}
          path: ${{ runner.temp }}/${{ env.changed_assets_artifact }}
      
      - name: Upload non-testable assets
        uses: actions/upload-artifact@v3
        if: steps.find-updated-assets.outputs.updated_count > 0
        with:
          name: ${{ env.releasable_assets_artifact }}
          path: |
            ${{ runner.temp }}/${{ env.changed_assets_artifact }}
            !${{ runner.temp }}/${{ env.changed_assets_artifact }}/environment

    outputs:
      updated_env_count: ${{ steps.find-updated-assets.outputs.updated_env_count }}

  build-images:
    name: Build and test images
    if: needs.identify-updated-assets.outputs.updated_env_count > 0
    runs-on: ubuntu-latest
    needs: identify-updated-assets
    environment: BuiltinRegistryDev

    env:
      build_logs_artifact_name: build-logs
      tested_dir: tested-environments

    steps:
      - name: Download changed assets
        uses: actions/download-artifact@v3
        with:
          name: ${{ env.changed_assets_artifact }}
          path: ${{ runner.temp }}/${{ env.changed_assets_artifact }}
    
      - name: Clone branch
        uses: actions/checkout@v3
        with:
          fetch-depth: 1
          path: ${{ env.main_dir }}
      
      - name: Use Python 3.8 or newer
        uses: actions/setup-python@v3
        with:
          python-version: '>=3.8'
    
      - name: Install dependencies
        run: pip install -e $main_dir/$scripts_azureml_assets_dir

      - name: Log in to Azure
        uses: azure/login@v1
        with:
          creds: ${{ secrets.AZ_CREDS_BUILTINREGISTRYDEV }}
      
      - name: Build and test images
        id: build-images
        run: python -u $main_dir/$scripts_environment_dir/build.py -i '${{ runner.temp }}'/$changed_assets_artifact -a $asset_config_filename -o '${{ runner.temp }}'/$tested_dir -l '${{ runner.temp }}'/$build_logs_artifact_name -g ${{ secrets.RESOURCE_GROUP }} -r ${{ secrets.BUILD_CONTAINER_REGISTRY }} -T 'python -V'
        continue-on-error: true
      
      - name: Upload build logs
        uses: actions/upload-artifact@v3
        if: always()
        with:
          name: ${{ env.build_logs_artifact_name }}
          path: ${{ runner.temp }}/${{ env.build_logs_artifact_name }}
      
      - name: Upload tested assets
        if: steps.build-images.outputs.success_count > 0
        uses: actions/upload-artifact@v3
        with:
          name: ${{ env.releasable_assets_artifact }}
          path: ${{ runner.temp }}/${{ env.tested_dir }}

  check-release-assets:
    name: Check releaseable assets
    if: always()
    runs-on: ubuntu-latest
    needs:
      - identify-updated-assets
      - build-images

    steps:
      - name: Download releasable assets
        id: download-artifact
        uses: actions/download-artifact@v3
        with:
          name: ${{ env.releasable_assets_artifact }}
          path: ${{ runner.temp }}/${{ env.releasable_assets_artifact }}
        continue-on-error: true

    outputs:
      has_releasable_assets: ${{ steps.download-artifact.outputs.download-path }}

  release-assets:
    name: Release assets
    if: always() && github.ref == 'refs/heads/main'
    runs-on: ubuntu-latest
    needs: check-release-assets
    
    steps:
      - name: Download releasable assets
        if: needs.check-release-assets.outputs.has_releasable_assets != ''
        id: download-artifact
        uses: actions/download-artifact@v3
        with:
          name: ${{ env.releasable_assets_artifact }}
          path: ${{ runner.temp }}/${{ env.releasable_assets_artifact }}
      
      - name: Clone branch
        uses: actions/checkout@v3
        with:
          fetch-depth: 1
          path: ${{ env.main_dir }}

      - name: Clone release branch
        uses: actions/checkout@v3
        if: steps.download-artifact.outputs.download-path
        with:
          ref: release
          fetch-depth: 0
          path: ${{ env.release_dir }}

      # TODO: Not needed unless calling update_assets.py
      - name: Use Python 3.8 or newer
        uses: actions/setup-python@v3
        with:
          python-version: '>=3.8'
    
      - name: Install dependencies
        run: pip install -e $main_dir/$scripts_azureml_assets_dir
      
      # TODO: This is overkill, just copy directories without involving this script
      - name: Update release branch
        run: python -u $main_dir/$scripts_assets_dir/update_assets.py -i ${{ runner.temp }}/$releasable_assets_artifact -r $release_dir -c
  
      - name: Convert test files and scripts
        run: python -u $main_dir/$scripts_assets_dir/release/test_file_convert.py -i $main_dir/training -r $release_dir -a training

      - name: Copy e2e test scripts
        run: | 
          if [ -d $release_dir/scripts ]; then 
            rm -Rf $release_dir/scripts 
          fi
          mkdir -p $release_dir/scripts
          cp -r $main_dir/$scripts_azureml_assets_dir $release_dir/scripts

      - name: Create commit and push
        run: |
          if [[ -z $(git status -s) ]]; then
            echo "No updates to the release branch"
            exit 0
          fi
          git config user.email "<>"
          git config user.name "$GITHUB_WORKFLOW"
          git add -A .
          git commit -m "Release $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"
          git push
        working-directory: ${{ env.release_dir }}