schedules:
  # Cron string < minute hour day-of-month month day-of-week>
  #             * means all like '*' in day of month means everyday
  # Run only on main branch
  # 'always' controls whether to run only if there is a change or not
  # Run this pipeline every 15:00 time
  - cron: '0 15 * * 6'
    displayName: 'Weekly Perf Blobfuse2 Perf Run'
    branches:
      include:
        - main
parameters:
  - name: resnet_test
    displayName: 'ResNet50 Test'
    type: boolean
    default: true


stages:
- stage: ShortRunning
  jobs:
    - job: PerformanceEval
      timeoutInMinutes: 240
      strategy:
        matrix:
          Ubuntu-20:
            DistroVer: "Ubn20_PERF"
            AgentName: "blobfuse-ubuntu20"
            Description: "Blobfuse2 Perf Test"

      pool:
        name: "blobfuse-perf-pool"
        demands:
          - ImageOverride -equals $(AgentName)

      variables:
        - group: NightlyBlobFuse
        - name: MOUNT_DIR
          value: "/home/vsts/workv2/blobfuse2mnt"
        - name: TEMP_DIR
          value: "/home/vsts/workv2/blobfuse2tmp"
        - name: BLOBFUSE2_CFG
          value: "$(System.DefaultWorkingDirectory)/blobfuse2_manual_perf.yaml"
        - name: BLOBFUSE_CFG
          value: "$(System.DefaultWorkingDirectory)/blobfuse_manual_perf.cfg"
        - name: GOPATH
          value: "/home/vsts/workv2/go"
        - name: ROOT_DIR
          value: "/home/vsts/workv2/"
        - name: WORK_DIR
          value: "/home/vsts/workv2/go/src/azure-storage-fuse"

      steps:
        - checkout: none      
        
        - script: |
            echo $(Description)
            hostnamectl
          displayName: 'Print Agent Info'
      
        - script: |          
            sudo apt-get update --fix-missing -o Dpkg::Options::="--force-confnew"
            sudo apt-get install fuse3 make cmake gcc g++ python3-setuptools python3-pip parallel fio -y -o Dpkg::Options::="--force-confnew"
          displayName: 'Install libraries'

        - script: |          
            sudo apt-get install blobfuse -y -o Dpkg::Options::="--force-confnew"
          displayName: 'Install blobfuse'

          # Prestart cleanup
        - script: |
            sudo fusermount3 -u $(MOUNT_DIR)
            sudo kill -9 `pidof blobfuse2`
            sudo rm -rf $(ROOT_DIR)
          displayName: 'PreBuild Cleanup'

        # Create directory structure
        - script: |
            sudo rm -rf $(ROOT_DIR)
            sudo mkdir -p $(ROOT_DIR)
            sudo chown -R `whoami` $(ROOT_DIR)
            chmod 777 $(ROOT_DIR)
            mkdir -p $(ROOT_DIR)/go/src
          displayName: 'Create Directory Structure'
    
        # Checkout the code  
        - script: |
            git clone https://github.com/Azure/azure-storage-fuse
          displayName: 'Checkout Code'
          workingDirectory: $(ROOT_DIR)/go/src

        # Pull the branch
        - script: |
            git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`
          displayName: 'Checkout Branch'
          workingDirectory: $(WORK_DIR)

        # -------------------------------------------------------
        # Pull and build the code
        - template: 'azure-pipeline-templates/build.yml'
          parameters:
            working_directory: $(WORK_DIR)
            root_dir: $(ROOT_DIR)
            mount_dir: $(MOUNT_DIR)
            temp_dir: $(TEMP_DIR)
            gopath: $(GOPATH)
            container: cont1
            skip_ut: true

        - script: |
            cd $(WORK_DIR)
            $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key_perf.yaml --container-name=cont1 --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG)
            cat $(BLOBFUSE2_CFG)
          displayName: "Generate v2 Config File"
          env:
            NIGHTLY_STO_ACC_NAME: $(PERF_WEEKLY_STO_BLOB_ACC_NAME)
            NIGHTLY_STO_ACC_KEY: $(PERF_WEEKLY_STO_BLOB_ACC_KEY)
            ACCOUNT_TYPE: 'block'
            ACCOUNT_ENDPOINT: 'https://$(PERF_WEEKLY_STO_BLOB_ACC_NAME).blob.core.windows.net'
          continueOnError: false
        
        - script: |
            touch $(BLOBFUSE_CFG)
            echo "accountName $(PERF_WEEKLY_STO_BLOB_ACC_NAME)" >> $(BLOBFUSE_CFG)
            echo "accountKey $(PERF_WEEKLY_STO_BLOB_ACC_KEY)" >> $(BLOBFUSE_CFG)
            echo "authType Key" >> $(BLOBFUSE_CFG)
            echo "containerName cont1" >> $(BLOBFUSE_CFG)
            cat $(BLOBFUSE_CFG)
          displayName: "Generate v1 Config File"
          continueOnError: false

        # --------------------------------------------------------------------------------------------
        # FIO Tests
        - script: |
            chmod 777 ./test/scripts/fio.sh
            ./test/scripts/fio.sh $(MOUNT_DIR)/fio $(TEMP_DIR) $(BLOBFUSE2_CFG) $(BLOBFUSE_CFG) rw
          displayName: 'FIO Sequential Test'
          workingDirectory: $(WORK_DIR)

        - script: |
            cat results_fio_rw.txt
          displayName: 'FIO Sequential Results'
          workingDirectory: $(WORK_DIR)

        - script: |
            chmod 777 ./test/scripts/fio.sh
            ./test/scripts/fio.sh $(MOUNT_DIR)/fio $(TEMP_DIR) $(BLOBFUSE2_CFG) $(BLOBFUSE_CFG) randrw
          displayName: 'FIO Random Test'
          workingDirectory: $(WORK_DIR)

        - script: |
            cat results_fio_randrw.txt
          displayName: 'FIO Random Results'
          workingDirectory: $(WORK_DIR)

        - script: |
            chmod 777 ./test/scripts/fio.sh
            ./test/scripts/fio.sh $(MOUNT_DIR)/fio $(TEMP_DIR) $(BLOBFUSE2_CFG) $(BLOBFUSE_CFG) rw csi
          displayName: 'FIO CSI Test'
          workingDirectory: $(WORK_DIR)

        - script: |
            cat results_fio_rw.txt
          displayName: 'FIO CSI Results'
          workingDirectory: $(WORK_DIR)

        # --------------------------------------------------------------------------------------------
        # Upload-DownloadFIO Tests
        - script: |
            chmod 777 ./test/scripts/run.sh
            ./test/scripts/run.sh $(MOUNT_DIR)/run $(TEMP_DIR) $(BLOBFUSE2_CFG) $(BLOBFUSE_CFG)

          displayName: 'Upload Download'
          workingDirectory: $(WORK_DIR)

        - script: |
            cat results.txt
          displayName: 'Upload Download Results'
          workingDirectory: $(WORK_DIR)

        # --------------------------------------------------------------------------------------------
        # Git clone test
        - script: |
            chmod 777 ./test/scripts/git_clone.sh
            ./test/scripts/git_clone.sh $(MOUNT_DIR)/git $(TEMP_DIR) $(BLOBFUSE2_CFG) $(BLOBFUSE_CFG)

          displayName: 'Git Clone'
          workingDirectory: $(WORK_DIR)

        - script: |
            cat results_git_clone.txt
          displayName: 'Git Clone Results'
          workingDirectory: $(WORK_DIR)
        
- ${{ if eq(parameters.resnet_test, true) }}:
  - stage: LongRunning
    dependsOn: ShortRunning
    jobs:
      - job: ResNet50
        timeoutInMinutes: 2800 # two day timeout
        strategy:
          matrix:
            Ubuntu-20:
              DistroVer: "Ubn20_PERF"
              AgentName: "blobfuse-ubuntu20"
              Description: "Blobfuse2 Perf Test"

        pool:
          name: "blobfuse-perf-pool"
          demands:
            - ImageOverride -equals $(AgentName)

        variables:
          - group: NightlyBlobFuse
          - name: MOUNT_DIR
            value: "/home/vsts/workv2/blob_mnt"
          - name: TEMP_DIR
            value: "/home/vsts/workv2/blobfuse2tmp"
          - name: BLOBFUSE2_CFG
            value: "$(System.DefaultWorkingDirectory)/blobfuse2_manual_perf.yaml"
          - name: GOPATH
            value: "/home/vsts/workv2/go"
          - name: ROOT_DIR
            value: "/home/vsts/workv2/"
          - name: WORK_DIR
            value: "/home/vsts/workv2/go/src/azure-storage-fuse"

        steps:
          - checkout: none

          - script: |          
              sudo apt-get update --fix-missing -o Dpkg::Options::="--force-confnew"
              sudo apt-get install fuse3 make cmake gcc g++ python3-setuptools python3-pip parallel fio -y -o Dpkg::Options::="--force-confnew"
            displayName: 'Install libraries'

            # Prestart cleanup
          - script: |
              sudo fusermount3 -u $(MOUNT_DIR)
              sudo kill -9 `pidof blobfuse2`
              sudo rm -rf $(ROOT_DIR)
            displayName: 'PreBuild Cleanup'

          # Create directory structure
          - script: |
              sudo rm -rf $(ROOT_DIR)
              sudo mkdir -p $(ROOT_DIR)
              sudo chown -R `whoami` $(ROOT_DIR)
              chmod 777 $(ROOT_DIR)
              mkdir -p $(ROOT_DIR)/go/src
            displayName: 'Create Directory Structure'
      
          # Checkout the code  
          - script: |
              git clone https://github.com/Azure/azure-storage-fuse
            displayName: 'Checkout Code'
            workingDirectory: $(ROOT_DIR)/go/src

          # Pull the branch
          - script: |
              git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`
            displayName: 'Checkout Branch'
            workingDirectory: $(WORK_DIR)

          # -------------------------------------------------------
          # Pull and build the code
          - template: 'azure-pipeline-templates/build.yml'
            parameters:
              working_directory: $(WORK_DIR)
              root_dir: $(ROOT_DIR)
              mount_dir: $(MOUNT_DIR)
              temp_dir: $(TEMP_DIR)
              gopath: $(GOPATH)
              container: cont1
              skip_ut: true

          - script: |
              cd $(ROOT_DIR)
              pip install numpy tensorflow Pillow
            displayName: "Install Python Dependencies"
            continueOnError: false

          - script: |
              cd $(WORK_DIR)
              $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key_perf.yaml --container-name=cont1 --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG)
              cat $(BLOBFUSE2_CFG)
            displayName: "Generate v2 Config File"
            env:
              NIGHTLY_STO_ACC_NAME: $(PERF_WEEKLY_STO_BLOB_ACC_NAME)
              NIGHTLY_STO_ACC_KEY: $(PERF_WEEKLY_STO_BLOB_ACC_KEY)
              ACCOUNT_TYPE: 'block'
              ACCOUNT_ENDPOINT: 'https://$(PERF_WEEKLY_STO_BLOB_ACC_NAME).blob.core.windows.net'
            continueOnError: false
            
          - script: |
              cd $(ROOT_DIR)
              wget https://github.com/Azure/azure-storage-fuse/releases/download/blobfuse2-2.0.0-preview.1/blobfuse2-2.0.0-preview.1-ubuntu-20.04-x86-64.deb
              sudo dpkg -i $(ROOT_DIR)/blobfuse2-2.0.0-preview.1-ubuntu-20.04-x86-64.deb
              sudo apt-get install -f
              sudo apt-get install fuse3
              blobfuse2 mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) --default-working-dir=$(WORK_DIR)
              sleep 10
              cd $(WORK_DIR)
              python3 $(WORK_DIR)/test/perf_test/resnet50_classify.py --dataset='$(MOUNT_DIR)/data' --job='binary' --log=$(ROOT_DIR)/blobfuse2-perf.json
            displayName: "Blobfuse2 ResNet50 Binary"
            continueOnError: false

          - script: |
              sudo fusermount3 -u ${MOUNT_DIR}
              sudo kill -9 `pidof blobfuse2` || true
            displayName: "Unmount Blobfuse2 Binary Run"

          - script: |
              cd $(WORK_DIR)
              $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key.yaml --container-name=cont1 --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG)
              $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) --default-working-dir=$(WORK_DIR)
              sleep 10
              python3 $(WORK_DIR)/test/perf_test/resnet50_classify.py --dataset='$(MOUNT_DIR)/data' --job='main' --log=$(ROOT_DIR)/blobfuse2-perf.json
            displayName: "Blobfuse2 ResNet50 on Main"
            env:
              NIGHTLY_STO_ACC_NAME: $(PERF_WEEKLY_STO_BLOB_ACC_NAME)
              NIGHTLY_STO_ACC_KEY: $(PERF_WEEKLY_STO_BLOB_ACC_KEY)
              ACCOUNT_TYPE: 'block'
              ACCOUNT_ENDPOINT: 'https://$(PERF_WEEKLY_STO_BLOB_ACC_NAME).blob.core.windows.net'
              VERBOSE_LOG: false
              USE_HTTP: false
            continueOnError: false

          - script: |
              cd $(WORK_DIR)
              python3 $(WORK_DIR)/test/perf_test/generate_perf_report.py --metrics=images/second --log=$(ROOT_DIR)/blobfuse2-perf.json
            displayName: "Perf Regression Test"
            continueOnError: false

          - publish: $(ROOT_DIR)/blobfuse2-perf.json
            artifact: Blobfuse2_performance_report
            displayName: Publish Performance Report

          - script: |
              sudo fusermount3 -u ${MOUNT_DIR}
              sudo kill -9 `pidof blobfuse2` || true
            displayName: "Unmount Blobfuse2 Main Branch Run"

          # Cleanup
          - template: 'azure-pipeline-templates/cleanup.yml'
            parameters:
              working_dir: $(WORK_DIR)
              mount_dir: $(MOUNT_DIR)
              temp_dir: $(TEMP_DIR)