azure-storage-fuse/blobfuse2-perf.yaml

schedules:
  # Cron string < minute hour day-of-month month day-of-week>
  #             * means all like '*' in day of month means everyday
  # Run only on main branch
  # 'always' controls whether to run only if there is a change or not
  # Run this pipeline every 15:00 time
  - cron: '0 15 * * 6'
    displayName: 'Weekly Perf Blobfuse2 Perf Run'
    branches:
      include:
        - main

jobs:
  - job: Blobfuse2_Perf_Test
    timeoutInMinutes: 2800 # two day timeout
    strategy:
      matrix:
        Ubuntu-20:
          DistroVer: "Ubn20_PERF"
          AgentName: "UBN20-PERF"
          Description: "Blobfuse2 Perf Test"

    pool:
      name: "Blobfuse Pool"
      demands:
        - Agent.Name -equals $(AgentName)

    variables:
      - group: NightlyBlobFuse
      - name: MOUNT_DIR
        value: "/home/vsts/workv2/blob_mnt"
      - name: TEMP_DIR
        value: "/home/vsts/workv2/blobfuse2tmp"
      - name: BLOBFUSE2_CFG
        value: "/home/tamer/blobfuse2.yaml"
      - name: GOPATH
        value: "/home/vsts/workv2/go"
      - name: ROOT_DIR
        value: "/home/vsts/workv2/"
      - name: WORK_DIR
        value: "/home/vsts/workv2/go/src/azure-storage-fuse"

    steps:
      - checkout: none

        # Prestart cleanup
      - script: |
          sudo fusermount3 -u $(MOUNT_DIR)
          sudo kill -9 `pidof blobfuse2`
          sudo rm -rf $(ROOT_DIR)
        displayName: 'PreBuild Cleanup'

      # Create directory structure
      - script: |
          sudo rm -rf $(ROOT_DIR)
          sudo mkdir -p $(ROOT_DIR)
          sudo chown -R `whoami` $(ROOT_DIR)
          chmod 777 $(ROOT_DIR)
          mkdir -p $(ROOT_DIR)/go/src
        displayName: 'Create Directory Structure'
  
      # Checkout the code  
      - script: |
          git clone https://github.com/Azure/azure-storage-fuse
        displayName: 'Checkout Code'
        workingDirectory: $(ROOT_DIR)/go/src

      # Pull the branch
      - script: |
          git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`
        displayName: 'Checkout Branch'
        workingDirectory: $(WORK_DIR)

      # -------------------------------------------------------
      # Pull and build the code
      - template: 'azure-pipeline-templates/build.yml'
        parameters:
          working_directory: $(WORK_DIR)
          root_dir: $(ROOT_DIR)
          mount_dir: $(MOUNT_DIR)
          temp_dir: $(TEMP_DIR)
          gopath: $(GOPATH)
          container: cont1
          skip_ut: true

      - script: |
          cd $(ROOT_DIR)
          pip install numpy tensorflow
        displayName: "Install Python Dependencies"
        continueOnError: false

      - script: |
          cd $(ROOT_DIR)
          wget https://github.com/Azure/azure-storage-fuse/releases/download/blobfuse2-2.0.0-preview.1/blobfuse2-2.0.0-preview.1-ubuntu-20.04-x86-64.deb
          sudo dpkg -i $(ROOT_DIR)/blobfuse2-2.0.0-preview.1-ubuntu-20.04-x86-64.deb
          sudo apt-get install -f
          sudo apt-get install fuse3
          blobfuse2 mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) --default-working-dir=$(WORK_DIR)
          cd $(WORK_DIR)
          python3 $(WORK_DIR)/test/perf_test/resnet50_classify.py --dataset='$(MOUNT_DIR)/data' --job='binary' --log=$(ROOT_DIR)/blobfuse2-perf.json
        displayName: "Blobfuse2 ResNet50 Binary"
        continueOnError: false

      - script: |
          sudo fusermount3 -u ${MOUNT_DIR}
          sudo kill -9 `pidof blobfuse2` || true
        displayName: "Unmount Blobfuse2 Binary Run"

      - script: |
          cd $(WORK_DIR)
          $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key.yaml --container-name=cont1 --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG)
          $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) --default-working-dir=$(WORK_DIR)
          python3 $(WORK_DIR)/test/perf_test/resnet50_classify.py --dataset='$(MOUNT_DIR)/data' --job='main' --log=$(ROOT_DIR)/blobfuse2-perf.json
        displayName: "Blobfuse2 ResNet50 on Main"
        env:
          NIGHTLY_STO_ACC_NAME: $(PERF_WEEKLY_STO_BLOB_ACC_NAME)
          NIGHTLY_STO_ACC_KEY: $(PERF_WEEKLY_STO_BLOB_ACC_KEY)
          ACCOUNT_TYPE: 'block'
          ACCOUNT_ENDPOINT: 'https://$(PERF_WEEKLY_STO_BLOB_ACC_NAME).blob.core.windows.net'
          VERBOSE_LOG: false
        continueOnError: false

      - script: |
          cd $(WORK_DIR)
          python3 $(WORK_DIR)/test/perf_test/generate_perf_report.py --metrics=images/second --log=$(ROOT_DIR)/blobfuse2-perf.json
        displayName: "Perf Regression Test"
        continueOnError: false

      - publish: $(ROOT_DIR)/blobfuse2-perf.json
        artifact: Blobfuse2_performance_report
        displayName: Publish Performance Report

      - script: |
          sudo fusermount3 -u ${MOUNT_DIR}
          sudo kill -9 `pidof blobfuse2` || true
        displayName: "Unmount Blobfuse2 Main Branch Run"

      # Cleanup
      - template: 'azure-pipeline-templates/cleanup.yml'
        parameters:
          working_dir: $(WORK_DIR)
          mount_dir: $(MOUNT_DIR)
          temp_dir: $(TEMP_DIR)
      
      - script: |
          sudo rm -rf ${ROOT_DIR}
          pwd
          cd /`pwd | cut -d '/' -f 2,3,4,5`
          sudo rm -rf [0-9]
        displayName: 'Clean Agent Directories'
        condition: always()