azure-storage-fuse/blobfuse2-nightly.yaml

# Blobfuse2 Nightly Build Pipeline

schedules:
  # Cron string < minute hour day-of-month month day-of-week>
  #             * means all, for example '*' in day of month means everyday
  # Run only on main branch
  # 'always' controls whether to run only if there is a change or not
  # Run this pipeline every 15:00 time
  - cron: '0 15 * * *'
    displayName: 'Daily midnight Blobfuse2 build'
    branches:
      include:
        - main

parameters:
  - name: base_test
    # Quick test or an exhaustive test
    displayName: 'Basic test'
    type: boolean
    default: true

  - name: exhaustive_test
    # Quick test or an exhaustive test
    displayName: 'Exhaustive test'
    type: boolean
    default: true
  
  - name: proxy_test
  # Proxy tests
    displayName: 'Proxy test'
    type: boolean
    default: true

  - name: msi_test
    # MSI auth based test suites to be run or not
    displayName: 'MSI test'
    type: boolean
    default: true

  - name: healthmon_test
    displayName: 'Healthmon test'
    type: boolean
    default: true

  - name: quick_stress
    displayName: 'Quick Stress'
    type: boolean
    default: true
  
  - name: data_validation
    displayName: 'Data Validation Test'
    type: boolean
    default: true

  - name: verbose_log
    displayName: 'Verbose Log'
    type: boolean
    default: false


stages:
  - ${{ if eq(parameters.base_test, true) }}:
    - stage: BuildAndTest
      jobs:
        # Ubuntu Tests
        - job: Set_1
          timeoutInMinutes: 300

          strategy:
            matrix:
              Ubuntu-20-BlockBlob:
                imageName: 'ubuntu-20.04'
                containerName: 'test-cnt-ubn-20'
                adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_20)
                fuselib: 'libfuse-dev'
                tags: 'fuse2'
              Ubuntu-22-BlockBlob:
                imageName: 'ubuntu-22.04'
                containerName: 'test-cnt-ubn-22'
                adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22)
                fuselib: 'libfuse3-dev'
                tags: 'fuse3'
          pool:
            vmImage: $(imageName)

          variables:
            - group: NightlyBlobFuse
            - name: MOUNT_DIR
              value: '$(Pipeline.Workspace)/blob_mnt'
            - name: TEMP_DIR
              value: '$(Pipeline.Workspace)/blobfuse2_tmp'
            - name: BLOBFUSE2_CFG
              value: '$(Pipeline.Workspace)/blobfuse2.yaml'
            - name: BLOBFUSE2_SAS_CFG
              value: '$(Pipeline.Workspace)/blobfuse2_sas_config.yaml'
            - name: BLOBFUSE2_SPN_CFG
              value: '$(Pipeline.Workspace)/blobfuse2_spn_config.yaml'
            - name: BLOBFUSE2_STREAM_CFG
              value: '$(Pipeline.Workspace)/blobfuse2_stream.yaml'
            - name: BLOBFUSE2_STREAM_FILENAME_CFG
              value: '$(Pipeline.Workspace)/blobfuse2_stream_filename.yaml'
            - name: BLOBFUSE2_ADLS_CFG
              value: '$(Pipeline.Workspace)/blobfuse2.adls.yaml'
            - name: BLOBFUSE2_GTEST_CFG
              value: '$(Pipeline.Workspace)/connection.yaml'
            - name: BLOBFUSE2_AZURITE_CFG
              value: '$(Pipeline.Workspace)/blobfuse2_azurite_config.yaml'
            - name: BLOBFUSE2_STRESS_DIR
              value: '$(Pipeline.Workspace)/blobfuse2_stress'
            - name: DECODE_PERCENTS
              value: false
            - name: GOPATH
              value: '$(Pipeline.Workspace)/go'
            - name: ROOT_DIR
              value: '$(System.DefaultWorkingDirectory)' 
            - name: WORK_DIR
              value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse'

          steps:
            - checkout: none

            # Clone the repo
            - script: |
                git clone https://github.com/Azure/azure-storage-fuse
              displayName: 'Checkout Code'
              workingDirectory: $(ROOT_DIR)

            # Checkout the branch
            - script: |
                git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`
              displayName: 'Checkout Branch'
              workingDirectory: $(WORK_DIR)

            - script: |
                sudo apt-get update --fix-missing
                sudo apt-get install $(fuselib) -y
              displayName: 'Install libfuse'

            # -------------------------------------------------------
            # Pull and build the code
            - template: 'azure-pipeline-templates/build.yml'
              parameters:
                working_directory: $(WORK_DIR)
                root_dir: $(Pipeline.Workspace)
                mount_dir: $(MOUNT_DIR)
                temp_dir: $(TEMP_DIR)
                gopath: $(GOPATH)
                container: $(containerName)
                tags: $(tags)
                fuselib: $(fuselib)

            # -------------------------------------------------------
            - template: 'azure-pipeline-templates/invalid-command-tests.yml'
              parameters:
                working_dir: $(WORK_DIR)
                mount_dir: $(MOUNT_DIR)

            - ${{ if eq(parameters.exhaustive_test, true) }}:    
              - template: 'azure-pipeline-templates/verbose-tests.yml'
                parameters:
                  service: 'BlockBlob'
                  account_type: 'block'
                  account_endpoint: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net' 
                  adls: false
                  account_name: $(NIGHTLY_STO_BLOB_ACC_NAME)
                  account_key: $(NIGHTLY_STO_BLOB_ACC_KEY)
                  account_sas: $(NIGHTLY_STO_ACC_SAS)
                  spn_account_name: $(AZTEST_BLOCK_ACC_NAME)
                  spn_account_endpoint: 'https://$(AZTEST_BLOCK_ACC_NAME).blob.core.windows.net' 
                  client_id: $(AZTEST_CLIENT)
                  tenant_id: $(AZTEST_TENANT)
                  client_secret: $(AZTEST_SECRET)
                  container: $(containerName)
                  config: $(BLOBFUSE2_CFG)
                  working_dir: $(WORK_DIR)
                  mount_dir: $(MOUNT_DIR)
                  temp_dir: $(TEMP_DIR)
                  stress_dir: $(BLOBFUSE2_STRESS_DIR)
                  huge_container: 'testcnt1'
                  quick_stress: ${{ parameters.quick_stress }}
                  test_key_credential: true 
                  test_sas_credential: true
                  test_spn_credential: true
                  test_stream: true
                  test_azurite: true
                  stream_config: $(BLOBFUSE2_STREAM_CFG)
                  stream_filename_config: $(BLOBFUSE2_STREAM_FILENAME_CFG)
                  sas_credential_config: $(BLOBFUSE2_SAS_CFG)
                  spn_credential_config: $(BLOBFUSE2_SPN_CFG)
                  azurite_config: $(BLOBFUSE2_AZURITE_CFG)
                  distro_name: $(imageName)
                  verbose_log: ${{ parameters.verbose_log }}
                  tags: $(tags)

            - template: azure-pipeline-templates/cleanup.yml
              parameters:
                working_dir: $(WORK_DIR)
                mount_dir: $(MOUNT_DIR)
                temp_dir: $(TEMP_DIR)

        - job: Set_1_ARM
          timeoutInMinutes: 300

          strategy:
            matrix:
              Ubuntu-22-ARM64-BlockBlob:
                imageName: 'ubuntu-22.04'
                containerName: 'test-cnt-ubn-22'
                adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22)
                fuselib: 'libfuse3-dev'
                tags: 'fuse3'
          
          parameters:
            pool:
              vmImage: $(imageName)
            arch: arm64
            os: linux

          variables:
            - group: NightlyBlobFuse
            - name: MOUNT_DIR
              value: '$(Pipeline.Workspace)/blob_mnt'
            - name: TEMP_DIR
              value: '$(Pipeline.Workspace)/blobfuse2_tmp'
            - name: BLOBFUSE2_CFG
              value: '$(Pipeline.Workspace)/blobfuse2.yaml'
            - name: BLOBFUSE2_SAS_CFG
              value: '$(Pipeline.Workspace)/blobfuse2_sas_config.yaml'
            - name: BLOBFUSE2_SPN_CFG
              value: '$(Pipeline.Workspace)/blobfuse2_spn_config.yaml'
            - name: BLOBFUSE2_STREAM_CFG
              value: '$(Pipeline.Workspace)/blobfuse2_stream.yaml'
            - name: BLOBFUSE2_STREAM_FILENAME_CFG
              value: '$(Pipeline.Workspace)/blobfuse2_stream_filename.yaml'
            - name: BLOBFUSE2_ADLS_CFG
              value: '$(Pipeline.Workspace)/blobfuse2.adls.yaml'
            - name: BLOBFUSE2_GTEST_CFG
              value: '$(Pipeline.Workspace)/connection.yaml'
            - name: BLOBFUSE2_AZURITE_CFG
              value: '$(Pipeline.Workspace)/blobfuse2_azurite_config.yaml'
            - name: BLOBFUSE2_STRESS_DIR
              value: '$(Pipeline.Workspace)/blobfuse2_stress'
            - name: DECODE_PERCENTS
              value: false
            - name: GOPATH
              value: '$(Pipeline.Workspace)/go'
            - name: ROOT_DIR
              value: '$(System.DefaultWorkingDirectory)' 
            - name: WORK_DIR
              value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse'

          steps:
            - checkout: none

            - script: |
                hostnamectl
              displayName: 'Host info'
              workingDirectory: $(ROOT_DIR)

            # Clone the repo
            - script: |
                git clone https://github.com/Azure/azure-storage-fuse
              displayName: 'Checkout Code'
              workingDirectory: $(ROOT_DIR)

            # Checkout the branch
            - script: |
                git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`
              displayName: 'Checkout Branch'
              workingDirectory: $(WORK_DIR)

            - script: |
                sudo apt-get update --fix-missing
                sudo apt-get install $(fuselib) -y
              displayName: 'Install libfuse'

            # -------------------------------------------------------
            # Pull and build the code
            - template: 'azure-pipeline-templates/build.yml'
              parameters:
                working_directory: $(WORK_DIR)
                root_dir: $(Pipeline.Workspace)
                mount_dir: $(MOUNT_DIR)
                temp_dir: $(TEMP_DIR)
                gopath: $(GOPATH)
                container: $(containerName)
                tags: $(tags)
                fuselib: $(fuselib)

            # -------------------------------------------------------
            - template: 'azure-pipeline-templates/invalid-command-tests.yml'
              parameters:
                working_dir: $(WORK_DIR)
                mount_dir: $(MOUNT_DIR)

            - ${{ if eq(parameters.exhaustive_test, true) }}:    
              - template: 'azure-pipeline-templates/verbose-tests.yml'
                parameters:
                  service: 'BlockBlob'
                  account_type: 'block'
                  account_endpoint: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net' 
                  adls: false
                  account_name: $(NIGHTLY_STO_BLOB_ACC_NAME)
                  account_key: $(NIGHTLY_STO_BLOB_ACC_KEY)
                  account_sas: $(NIGHTLY_STO_ACC_SAS)
                  spn_account_name: $(AZTEST_BLOCK_ACC_NAME)
                  spn_account_endpoint: 'https://$(AZTEST_BLOCK_ACC_NAME).blob.core.windows.net' 
                  client_id: $(AZTEST_CLIENT)
                  tenant_id: $(AZTEST_TENANT)
                  client_secret: $(AZTEST_SECRET)
                  container: $(containerName)
                  config: $(BLOBFUSE2_CFG)
                  working_dir: $(WORK_DIR)
                  mount_dir: $(MOUNT_DIR)
                  temp_dir: $(TEMP_DIR)
                  stress_dir: $(BLOBFUSE2_STRESS_DIR)
                  huge_container: 'testcnt1'
                  quick_stress: ${{ parameters.quick_stress }}
                  test_key_credential: true 
                  test_sas_credential: true
                  test_spn_credential: true
                  test_stream: true
                  test_azurite: true
                  stream_config: $(BLOBFUSE2_STREAM_CFG)
                  stream_filename_config: $(BLOBFUSE2_STREAM_FILENAME_CFG)
                  sas_credential_config: $(BLOBFUSE2_SAS_CFG)
                  spn_credential_config: $(BLOBFUSE2_SPN_CFG)
                  azurite_config: $(BLOBFUSE2_AZURITE_CFG)
                  distro_name: $(imageName)
                  verbose_log: ${{ parameters.verbose_log }}
                  tags: $(tags)

            - template: azure-pipeline-templates/cleanup.yml
              parameters:
                working_dir: $(WORK_DIR)
                mount_dir: $(MOUNT_DIR)
                temp_dir: $(TEMP_DIR)

        # Ubuntu Tests
        - job: Set_2
          timeoutInMinutes: 300

          strategy:
            matrix:
              Ubuntu-20-ADLS:
                imageName: 'ubuntu-20.04'
                containerName: 'test-cnt-ubn-20'
                adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_20)
                fuselib: 'libfuse-dev'
                tags: 'fuse2'
              Ubuntu-22-ADLS:
                imageName: 'ubuntu-22.04'
                containerName: 'test-cnt-ubn-22'
                adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22)
                fuselib: 'libfuse3-dev'
                tags: 'fuse3'
          pool:
            vmImage: $(imageName)

          variables:
            - group: NightlyBlobFuse
            - name: MOUNT_DIR
              value: '$(Pipeline.Workspace)/blob_mnt'
            - name: TEMP_DIR
              value: '$(Pipeline.Workspace)/blobfuse2_tmp'
            - name: BLOBFUSE2_CFG
              value: '$(Pipeline.Workspace)/blobfuse2.yaml'
            - name: BLOBFUSE2_SAS_CFG
              value: '$(Pipeline.Workspace)/blobfuse2_sas_config.yaml'
            - name: BLOBFUSE2_SPN_CFG
              value: '$(Pipeline.Workspace)/blobfuse2_spn_config.yaml'
            - name: BLOBFUSE2_STREAM_CFG
              value: '$(Pipeline.Workspace)/blobfuse2_stream.yaml'
            - name: BLOBFUSE2_STREAM_FILENAME_CFG
              value: '$(Pipeline.Workspace)/blobfuse2_stream_filename.yaml'
            - name: BLOBFUSE2_ADLS_CFG
              value: '$(Pipeline.Workspace)/blobfuse2.adls.yaml'
            - name: BLOBFUSE2_GTEST_CFG
              value: '$(Pipeline.Workspace)/connection.yaml'
            - name: BLOBFUSE2_AZURITE_CFG
              value: '$(Pipeline.Workspace)/blobfuse2_azurite_config.yaml'
            - name: BLOBFUSE2_STRESS_DIR
              value: '$(Pipeline.Workspace)/blobfuse2_stress'
            - name: DECODE_PERCENTS
              value: false
            - name: GOPATH
              value: '$(Pipeline.Workspace)/go'
            - name: ROOT_DIR
              value: '$(System.DefaultWorkingDirectory)' 
            - name: WORK_DIR
              value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse'

          steps:
            - checkout: none

            # Clone the repo
            - script: |
                git clone https://github.com/Azure/azure-storage-fuse
              displayName: 'Checkout Code'
              workingDirectory: $(ROOT_DIR)

            # Checkout the branch
            - script: |
                git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`
              displayName: 'Checkout Branch'
              workingDirectory: $(WORK_DIR)

            - script: |
                sudo apt-get update --fix-missing
                sudo apt-get install $(fuselib) -y
              displayName: 'Install libfuse'

            # -------------------------------------------------------
            # Pull and build the code
            - template: 'azure-pipeline-templates/build.yml'
              parameters:
                working_directory: $(WORK_DIR)
                root_dir: $(Pipeline.Workspace)
                mount_dir: $(MOUNT_DIR)
                temp_dir: $(TEMP_DIR)
                gopath: $(GOPATH)
                container: $(containerName)
                tags: $(tags)
                fuselib: $(fuselib)
                skip_ut: true # Skip UT because Block Blob set runs it

            # -------------------------------------------------------
            - ${{ if eq(parameters.exhaustive_test, true) }}:    
              - template: 'azure-pipeline-templates/verbose-tests.yml'
                parameters:
                  service: 'ADLS'
                  account_type: 'adls'
                  account_endpoint: 'https://$(AZTEST_ADLS_ACC_NAME).dfs.core.windows.net' 
                  adls: true
                  account_name: $(AZTEST_ADLS_ACC_NAME)
                  account_key: $(AZTEST_ADLS_KEY)
                  account_sas: $(adlsSas)
                  spn_account_name: $(AZTEST_ADLS_ACC_NAME)
                  spn_account_endpoint: 'https://$(AZTEST_ADLS_ACC_NAME).dfs.core.windows.net' 
                  client_id: $(AZTEST_CLIENT)
                  tenant_id: $(AZTEST_TENANT)
                  client_secret: $(AZTEST_SECRET)
                  container: $(containerName)
                  config: $(BLOBFUSE2_ADLS_CFG)
                  working_dir: $(WORK_DIR)
                  mount_dir: $(MOUNT_DIR)
                  temp_dir: $(TEMP_DIR)
                  stress_dir: $(BLOBFUSE2_STRESS_DIR)
                  huge_container: 'testcnt'
                  quick_stress: ${{ parameters.quick_stress }}
                  test_key_credential: true
                  test_sas_credential: true
                  test_spn_credential: true
                  test_stream: true
                  test_azurite: false
                  stream_config: $(BLOBFUSE2_STREAM_CFG)
                  stream_filename_config: $(BLOBFUSE2_STREAM_FILENAME_CFG)
                  sas_credential_config: $(BLOBFUSE2_SAS_CFG)
                  spn_credential_config: $(BLOBFUSE2_SPN_CFG)
                  azurite_config: $(BLOBFUSE2_AZURITE_CFG)
                  distro_name: $(imageName)
                  verbose_log: ${{ parameters.verbose_log }}
                  tags: $(tags)

            - template: azure-pipeline-templates/cleanup.yml
              parameters:
                working_dir: $(WORK_DIR)
                mount_dir: $(MOUNT_DIR)
                temp_dir: $(TEMP_DIR)

        - ${{ if eq(parameters.proxy_test, true) }}:
            # -----------------------------------------------------------
            # Ubuntu-20.04 Proxy tests
          - job: Set_3
            timeoutInMinutes: 300
            strategy:
              matrix:
                Ubuntu-20-Proxy:
                  imageName: 'ubuntu-20.04'
                  containerName: 'test-cnt-ubn-18-proxy'
            pool:
              vmImage: $(imageName)

            variables:
              - group: NightlyBlobFuse
              - name: MOUNT_DIR
                value: '$(Pipeline.Workspace)/blob_mnt'
              - name: TEMP_DIR
                value: '$(Pipeline.Workspace)/blobfuse2_tmp'
              - name: BLOBFUSE2_CFG
                value: '$(Pipeline.Workspace)/blobfuse2_proxy.yaml'
              - name: BLOBFUSE2_ADLS_CFG
                value: '$(Pipeline.Workspace)/blobfuse2_proxy.adls.yaml'
              - name: DECODE_PERCENTS
                value: false
              - name: GOPATH
                value: '$(Pipeline.Workspace)/go'
              - name: ROOT_DIR
                value: '$(System.DefaultWorkingDirectory)' 
              - name: WORK_DIR
                value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse'

            steps:
              - checkout: none

              # Start Proxy
              - script: |
                  sudo apt-get update --fix-missing
                  sudo apt remove mitmproxy
                  sudo apt-get install python3-setuptools
                  sudo apt install python3-pip
                  sudo pip3 install mitmproxy
                  mitmdump -w proxy_logs.txt &
                displayName: 'Install & Start Proxy'

              # Configure Proxy cert & env
              - script: |
                  sudo mkdir /usr/local/share/ca-certificates/extra
                  sudo cp ~/.mitmproxy/mitmproxy-ca-cert.cer /usr/local/share/ca-certificates/extra/mitmproxy-ca-cert.crt
                  sudo update-ca-certificates
                  export https_proxy="127.0.0.1:8080"
                  export http_proxy="127.0.0.1:8080"
                displayName: 'Configure Proxy'

              # Clone the repo
              - script: |
                  git clone https://github.com/Azure/azure-storage-fuse
                displayName: 'Checkout Code'
                workingDirectory: $(ROOT_DIR)

              # Checkout the branch
              - script: |
                  git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`
                displayName: 'Checkout Branch'
                workingDirectory: $(WORK_DIR)

              - script: |
                  sudo apt-get update --fix-missing
                  sudo apt-get install libfuse3-dev
                displayName: 'Install fuse3'

              # -------------------------------------------------------
              # Pull, build and unit test the code
              - template: 'azure-pipeline-templates/build.yml'
                parameters:
                  working_directory: $(WORK_DIR)
                  root_dir: $(Pipeline.Workspace)
                  mount_dir: $(MOUNT_DIR)
                  temp_dir: $(TEMP_DIR)
                  gopath: $(GOPATH)
                  container: $(containerName)
                  proxy_address: "127.0.0.1:8080"
                  skip_ut: true

              # -------------------------------------------------------
              - script: |
                  cd $(WORK_DIR)
                  $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key_proxy.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG)
                displayName: 'Create Config File'
                env:
                  NIGHTLY_STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME)
                  NIGHTLY_STO_ACC_KEY: $(NIGHTLY_STO_BLOB_ACC_KEY)
                  ACCOUNT_TYPE: 'block'
                  ACCOUNT_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net'
                  VERBOSE_LOG: ${{ parameters.verbose_log }}
                continueOnError: false

              - script:
                  cat $(BLOBFUSE2_CFG)
                displayName: "Print config file"

              # --------------------------------------------------
              # End to End tests
              - template: 'azure-pipeline-templates/e2e-tests.yml'
                parameters:
                  working_dir: $(WORK_DIR)
                  mount_dir: $(MOUNT_DIR)
                  temp_dir: $(TEMP_DIR)
                  idstring: 'BlockBlob with Proxy and Key Credentials'
                  distro_name: $(imageName)
                  adls: false
                  artifact_name: 'blockblob_proxy_key.txt'
                  verbose_log: ${{ parameters.verbose_log }}
                  mountStep:
                    script: |
                      $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) --default-working-dir=$(WORK_DIR)
                    displayName: 'E2E TEST: Mount'
                    timeoutInMinutes: 3
                    continueOnError: false

              # --------------------------------------------------
              - script: |
                  cd $(WORK_DIR)
                  $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key_proxy.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_ADLS_CFG)
                displayName: 'Create ADLS Config File'
                env:
                  NIGHTLY_STO_ACC_NAME: $(AZTEST_ADLS_ACC_NAME)
                  NIGHTLY_STO_ACC_KEY: $(AZTEST_ADLS_KEY)
                  ACCOUNT_TYPE: 'adls'
                  ACCOUNT_ENDPOINT: 'https://$(AZTEST_ADLS_ACC_NAME).dfs.core.windows.net'
                  VERBOSE_LOG: ${{ parameters.verbose_log }}
                continueOnError: false

              - script:
                  cat $(BLOBFUSE2_ADLS_CFG)
                displayName: "Print ADLS config file"

              - template: 'azure-pipeline-templates/e2e-tests.yml'
                parameters:
                  working_dir: $(WORK_DIR)
                  mount_dir: $(MOUNT_DIR)
                  temp_dir: $(TEMP_DIR)
                  idstring: 'ADLS with Proxy and Key Credentials'
                  distro_name: $(imageName)
                  adls: true
                  artifact_name: 'adls_proxy_key.txt'
                  verbose_log: ${{ parameters.verbose_log }}
                  mountStep:
                    script: |
                      $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_ADLS_CFG) --default-working-dir=$(WORK_DIR)
                    displayName: 'FeatureTest ADLS: Mount'
                    timeoutInMinutes: 3
                    continueOnError: false
              # ------------------------------------------------------------
              # Auth Tests
            
              # Block SAS test
              - script: |
                  cd $(WORK_DIR)
                  $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_sas_proxy.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG)
                displayName: "Create SAS Config File"
                env:
                  NIGHTLY_STO_BLOB_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME)
                  NIGHTLY_STO_ACC_SAS: $(NIGHTLY_STO_ACC_SAS)
                  ACCOUNT_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net'
                  VERBOSE_LOG: ${{ parameters.verbose_log }}
                continueOnError: false

              - script:
                  cat $(BLOBFUSE2_CFG)
                displayName: "Print SAS config file"

              - template: 'azure-pipeline-templates/verify-auth.yml'
                parameters:
                  working_dir: $(WORK_DIR)
                  mount_dir: $(MOUNT_DIR)
                  temp_dir: $(TEMP_DIR)
                  idstring: 'Block SAS'
                  distro_name: $(imageName)
                  mountStep:
                    script: |
                      $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) --default-working-dir=$(WORK_DIR)
                    displayName: 'AuthVerify-SAS: Mount'
                    continueOnError: false

              # Block SPN and proxy test
              - script: |
                  cd $(WORK_DIR)
                  $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_spn_proxy.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG)
                displayName: "Create SPN Config File"
                env:
                  NIGHTLY_SPN_ACC_NAME: $(AZTEST_BLOCK_ACC_NAME) 
                  NIGHTLY_SPN_CLIENT_ID: $(AZTEST_CLIENT)
                  NIGHTLY_SPN_TENANT_ID: $(AZTEST_TENANT)
                  NIGHTLY_SPN_CLIENT_SECRET: $(AZTEST_SECRET)
                  ACCOUNT_ENDPOINT: 'https://$(AZTEST_BLOCK_ACC_NAME).blob.core.windows.net'
                  VERBOSE_LOG: ${{ parameters.verbose_log }}
                continueOnError: false
                  
              - script:
                  cat $(BLOBFUSE2_CFG)
                displayName: "Print SPN Config File"

              - template: 'azure-pipeline-templates/verify-auth.yml'
                parameters:
                  working_dir: $(WORK_DIR)
                  mount_dir: $(MOUNT_DIR)
                  temp_dir: $(TEMP_DIR)
                  idstring: 'Block SPN'
                  distro_name: $(imageName)
                  mountStep:
                    script: |
                      $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) --default-working-dir=$(WORK_DIR)
                    displayName: 'AuthVerify-SPN: Mount'
                    continueOnError: false

              # ------------------------------------------------------------
              - template: 'azure-pipeline-templates/cleanup.yml'
                parameters:
                  working_dir: $(WORK_DIR)
                  mount_dir: $(MOUNT_DIR)
                  temp_dir: $(TEMP_DIR)
              - script: |
                  kill -9 $(pgrep mitmdump)
                displayName: 'Kill Proxy'
        
        - ${{ if eq(parameters.exhaustive_test, true) }}:
          # RHEL Tests
          - job: Set_4
            timeoutInMinutes: 60
            strategy:
              matrix:
                RHEL-7.5:
                  DistroVer: "RHEL-7.5"
                  Description: "Red Hat Enterprise Linux 7.5"
                  AgentName: "blobfuse-rhel7_5"
                  ContainerName: "test-cnt-rhel-75"
                  tags: 'fuse3'
                RHEL-8.6:
                  DistroVer: "RHEL-8.6"
                  Description: "Red Hat Enterprise Linux 8.6"
                  AgentName: "blobfuse-rhel8_6"
                  containerName: "test-cnt-rhel-86"
                  tags: 'fuse3'
            pool:
              name: "blobfuse-rhel-pool"
              demands:
                - ImageOverride -equals $(AgentName)

            variables:
              - group: NightlyBlobFuse
              - name: ROOT_DIR
                value: "/usr/pipeline/workv2"
              - name: WORK_DIR
                value: "/usr/pipeline/workv2/go/src/azure-storage-fuse"
              - name: skipComponentGovernanceDetection
                value: true
              - name: MOUNT_DIR
                value: "/usr/pipeline/workv2/blob_mnt"
              - name: TEMP_DIR
                value: "/usr/pipeline/workv2/temp"
              - name: BLOBFUSE2_CFG
                value: "/usr/pipeline/workv2/blobfuse2.yaml"
              - name: BLOBFUSE2_ADLS_CFG
                value: "/home/vsts/workv2/blobfuse2.adls.yaml"
              - name: GOPATH
                value: "/usr/pipeline/workv2/go"

            steps:
              # Go tool installer
              - task: GoTool@0
                inputs:
                  version: '1.20.5'
                displayName: "Install Go Version"

              - script: |
                  sudo touch /etc/yum.repos.d/centos.repo
                  sudo sh -c 'echo -e "[centos-extras]\nname=Centos extras - $basearch\nbaseurl=http://mirror.centos.org/centos/7/extras/x86_64\nenabled=1\ngpgcheck=1\ngpgkey=http://centos.org/keys/RPM-GPG-KEY-CentOS-7" > /etc/yum.repos.d/centos.repo'
                condition: eq(variables['AgentName'], 'blobfuse-rhel7_5')
                displayName: "Update OS mirrors"

              - template: 'azure-pipeline-templates/distro-tests.yml'
                parameters:
                  working_dir: $(WORK_DIR)
                  root_dir: $(ROOT_DIR)
                  temp_dir: $(TEMP_DIR)
                  mount_dir: $(MOUNT_DIR)
                  config_path: $(BLOBFUSE2_CFG)
                  container: $(ContainerName)
                  blob_account_name: $(NIGHTLY_STO_BLOB_ACC_NAME)
                  blob_account_key: $(NIGHTLY_STO_BLOB_ACC_KEY)
                  adls_account_name: $(AZTEST_ADLS_ACC_NAME)
                  adls_account_key: $(AZTEST_ADLS_KEY)
                  distro_name: $(AgentName)
                  gopath: $(GOPATH)
                  tags: $(tags) 
                  installStep:
                    script: |
                      sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo
                      sudo yum update -y
                      sudo yum groupinstall "Development Tools" -y
                      if [ $(AgentName) == "blobfuse-rhel7_5" ]; then
                        sudo yum-config-manager --save --setopt=rhui-rhel-7-server-dotnet-rhui-rpms.skip_if_unavailable=true
                        sudo yum install git fuse fuse3-libs fuse3-devel fuse3 rh-python36 -y
                      else
                        sudo yum install git fuse fuse3-libs fuse3-devel fuse3 python36 -y --nobest --allowerasing
                      fi
                    displayName: 'Install fuse'
                  verbose_log: ${{ parameters.verbose_log }}

          # Centos Tests
          - job: Set_5
            timeoutInMinutes: 60
            strategy:
              matrix:
                CentOS-7.9:
                  DistroVer: "CentOS-7.9"
                  Description: "CentOS 7.9"
                  AgentName: "blobfuse-centos7"
                  ContainerName: "test-cnt-cent-7"
                CentOS-8.5:
                  DistroVer: "CentOS-8.5"
                  Description: "CentOS 8.5"
                  AgentName: "blobfuse-centos8"
                  ContainerName: "test-cnt-cent-8"

            pool:
              name: "blobfuse-centos-pool"
              demands:
                - ImageOverride -equals $(AgentName)

            variables:
              - group: NightlyBlobFuse
              - name: ROOT_DIR
                value: "/usr/pipeline/workv2"
              - name: WORK_DIR
                value: "/usr/pipeline/workv2/go/src/azure-storage-fuse"
              - name: skipComponentGovernanceDetection
                value: true
              - name: MOUNT_DIR
                value: "/usr/pipeline/workv2/blob_mnt"
              - name: TEMP_DIR
                value: "/usr/pipeline/workv2/temp"
              - name: BLOBFUSE2_CFG
                value: "/usr/pipeline/workv2/blobfuse2.yaml"
              - name: BLOBFUSE2_ADLS_CFG
                value: "/home/vsts/workv2/blobfuse2.adls.yaml"
              - name: GOPATH
                value: "/usr/pipeline/workv2/go"

            steps:
              # Go tool installer
              - task: GoTool@0
                inputs:
                  version: '1.20.5'
                displayName: "Install Go Version"

              - script: |
                  sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-*
                  sudo sed -i 's|baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*
                condition: eq(variables['AgentName'], 'blobfuse-centos8')
                displayName: "Update OS mirrors"

              - template: 'azure-pipeline-templates/distro-tests.yml'
                parameters:
                  working_dir: $(WORK_DIR)
                  root_dir: $(ROOT_DIR)
                  temp_dir: $(TEMP_DIR)
                  mount_dir: $(MOUNT_DIR)
                  config_path: $(BLOBFUSE2_CFG)
                  container: $(ContainerName)
                  blob_account_name: $(NIGHTLY_STO_BLOB_ACC_NAME)
                  blob_account_key: $(NIGHTLY_STO_BLOB_ACC_KEY)
                  adls_account_name: $(AZTEST_ADLS_ACC_NAME)
                  adls_account_key: $(AZTEST_ADLS_KEY)
                  distro_name: $(AgentName)
                  gopath: $(GOPATH)
                  installStep:
                    script: |
                      sudo yum update -y --skip-broken
                      if [ $(AgentName) == "blobfuse-centos8" ]; then
                        sudo yum install gcc gcc-c++ make git fuse fuse3 fuse3-devel python36 -y --nobest --allowerasing
                      else
                        sudo yum install gcc gcc-c++ make git fuse3 fuse3-devel python36 -y
                      fi
                    displayName: 'Install fuse'
                  verbose_log: ${{ parameters.verbose_log }}

          # Oracle Tests
          - job: Set_6
            timeoutInMinutes: 60
            strategy:
              matrix:
                Oracle-8.1:
                  DistroVer: "Oracle-8.1"
                  Description: "Oracle Linux 8.1"
                  AgentName: "blobfuse-oracle81"
                  ContainerName: "test-cnt-ora-81"

            pool:
              name: "blobfuse-oracle-pool"
              demands:
                - ImageOverride -equals $(AgentName)

            variables:
              - group: NightlyBlobFuse
              - name: ROOT_DIR
                value: "/usr/pipeline/workv2"
              - name: WORK_DIR
                value: "/usr/pipeline/workv2/go/src/azure-storage-fuse"
              - name: skipComponentGovernanceDetection
                value: true
              - name: MOUNT_DIR
                value: "/usr/pipeline/workv2/blob_mnt"
              - name: TEMP_DIR
                value: "/usr/pipeline/workv2/temp"
              - name: BLOBFUSE2_CFG
                value: "/usr/pipeline/workv2/blobfuse2.yaml"
              - name: BLOBFUSE2_ADLS_CFG
                value: "/home/vsts/workv2/blobfuse2.adls.yaml"
              - name: GOPATH
                value: "/usr/pipeline/workv2/go"

            steps:
              # Go tool installer
              - task: GoTool@0
                inputs:
                  version: '1.20.5'
                displayName: "Install Go Version"

              - template: 'azure-pipeline-templates/distro-tests.yml'
                parameters:
                  working_dir: $(WORK_DIR)
                  root_dir: $(ROOT_DIR)
                  temp_dir: $(TEMP_DIR)
                  mount_dir: $(MOUNT_DIR)
                  config_path: $(BLOBFUSE2_CFG)
                  container: $(ContainerName)
                  blob_account_name: $(NIGHTLY_STO_BLOB_ACC_NAME)
                  blob_account_key: $(NIGHTLY_STO_BLOB_ACC_KEY)
                  adls_account_name: $(AZTEST_ADLS_ACC_NAME)
                  adls_account_key: $(AZTEST_ADLS_KEY)
                  distro_name: $(AgentName)
                  gopath: $(GOPATH)
                  installStep:
                    script: |
                      sudo yum update -y
                      sudo yum install gcc gcc-c++ make git fuse fuse3 fuse3-devel python36 -y --nobest --allowerasing 
                    displayName: 'Install fuse'
                  verbose_log: ${{ parameters.verbose_log }}

          # Debian Tests
          - job: Set_8
            timeoutInMinutes: 60
            strategy:
              matrix:
                Debian-10.0:
                  DistroVer: "Debian10.0"
                  Description: "Debian 10"
                  AgentName: "blobfuse-debian10"
                  ContainerName: "test-cnt-deb-10"
                  fuselib: 'fuse libfuse-dev'
                  tags: 'fuse2'
                Debian-11.0:
                  DistroVer: "Debian11.0"
                  Description: "Debian 11"
                  AgentName: "blobfuse-debian11"
                  ContainerName: "test-cnt-deb-11"
                  fuselib: 'fuse3 libfuse3-dev'
                  tags: 'fuse3'

            pool:
              name: "blobfuse-debian-pool"
              demands:
                - ImageOverride -equals $(AgentName)

            variables:
              - group: NightlyBlobFuse
              - name: ROOT_DIR
                value: "/usr/pipeline/workv2"
              - name: WORK_DIR
                value: "/usr/pipeline/workv2/go/src/azure-storage-fuse"
              - name: skipComponentGovernanceDetection
                value: true
              - name: MOUNT_DIR
                value: "/usr/pipeline/workv2/blob_mnt"
              - name: TEMP_DIR
                value: "/usr/pipeline/workv2/temp"
              - name: BLOBFUSE2_CFG
                value: "/usr/pipeline/workv2/blobfuse2.yaml"
              - name: BLOBFUSE2_ADLS_CFG
                value: "/home/vsts/workv2/blobfuse2.adls.yaml"
              - name: GOPATH
                value: "/usr/pipeline/workv2/go"

            steps:
              # Go tool installer
              - task: GoTool@0
                inputs:
                  version: '1.20.5'
                displayName: "Install Go Version"

              - template: 'azure-pipeline-templates/distro-tests.yml'
                parameters:
                  working_dir: $(WORK_DIR)
                  root_dir: $(ROOT_DIR)
                  temp_dir: $(TEMP_DIR)
                  mount_dir: $(MOUNT_DIR)
                  config_path: $(BLOBFUSE2_CFG)
                  container: $(ContainerName)
                  blob_account_name: $(NIGHTLY_STO_BLOB_ACC_NAME)
                  blob_account_key: $(NIGHTLY_STO_BLOB_ACC_KEY)
                  adls_account_name: $(AZTEST_ADLS_ACC_NAME)
                  adls_account_key: $(AZTEST_ADLS_KEY)
                  distro_name: $(AgentName)
                  tags: $(tags)
                  fuselib: $(fuselib)
                  gopath: $(GOPATH)
                  installStep:
                    script: |
                      sudo rm /etc/apt/sources.list.d/azure.list
                      sudo apt-get update --fix-missing -y
                      sudo apt-get install $(fuselib) -y
                      sudo apt-get install build-essential git python3 -y
                    displayName: 'Install fuse'
                  verbose_log: ${{ parameters.verbose_log }}

          # SUSE Tests
          - job: Set_9
            timeoutInMinutes: 60
            strategy:
              matrix:
                SUSE-15:
                  DistroVer: "SUSE-15"
                  Description: "SUSE Enterprise Linux 15"
                  AgentName: "blobfuse-suse15"
                  ContainerName: "test-cnt-suse-15"

            pool:
              name: "blobfuse-suse-pool"
              demands:
                - ImageOverride -equals $(AgentName)

            variables:
              - group: NightlyBlobFuse
              - name: ROOT_DIR
                value: "/usr/pipeline/workv2"
              - name: WORK_DIR
                value: "/usr/pipeline/workv2/go/src/azure-storage-fuse"
              - name: skipComponentGovernanceDetection
                value: true
              - name: MOUNT_DIR
                value: "/usr/pipeline/workv2/blob_mnt"
              - name: TEMP_DIR
                value: "/usr/pipeline/workv2/temp"
              - name: BLOBFUSE2_CFG
                value: "/usr/pipeline/workv2/blobfuse2.yaml"
              - name: BLOBFUSE2_ADLS_CFG
                value: "/home/vsts/workv2/blobfuse2.adls.yaml"
              - name: GOPATH
                value: "/usr/pipeline/workv2/go"

            steps:
              # Go tool installer
              - task: GoTool@0
                inputs:
                  version: '1.20.5'
                displayName: "Install Go Version"

              - template: 'azure-pipeline-templates/distro-tests.yml'
                parameters:
                  working_dir: $(WORK_DIR)
                  root_dir: $(ROOT_DIR)
                  temp_dir: $(TEMP_DIR)
                  mount_dir: $(MOUNT_DIR)
                  config_path: $(BLOBFUSE2_CFG)
                  container: $(ContainerName)
                  blob_account_name: $(NIGHTLY_STO_BLOB_ACC_NAME)
                  blob_account_key: $(NIGHTLY_STO_BLOB_ACC_KEY)
                  adls_account_name: $(AZTEST_ADLS_ACC_NAME)
                  adls_account_key: $(AZTEST_ADLS_KEY)
                  distro_name: $(AgentName)
                  gopath: $(GOPATH)
                  installStep:
                    script: |
                      sudo zypper -n install git golang make cmake gcc gcc-c++ glibc-devel fuse
                      wget https://rpmfind.net/linux/opensuse/distribution/leap/15.4/repo/oss/x86_64/fuse3-3.10.5-150400.1.7.x86_64.rpm
                      sudo zypper -n --no-gpg-checks install fuse3-3.10.5-150400.1.7.x86_64.rpm
                      fusermount3 --v
                      wget https://rpmfind.net/linux/opensuse/distribution/leap/15.4/repo/oss/x86_64/fuse3-devel-3.10.5-150400.1.7.x86_64.rpm
                      sudo zypper -n --no-gpg-checks install fuse3-devel-3.10.5-150400.1.7.x86_64.rpm 
                    displayName: 'Install fuse'
                  verbose_log: ${{ parameters.verbose_log }}

          # Mariner Tests
          - job: Set_10
            timeoutInMinutes: 60
            strategy:
              matrix:
                Mariner:
                  DistroVer: "Mariner"
                  Description: "CBL-Mariner Linux"
                  AgentName: "blobfuse-mariner"
                  ContainerName: "test-cnt-mari-1"
                  fuselib: 'libfuse-dev'
                  tags: 'fuse2'

            pool:
              name: "blobfuse-mariner-pool"
              demands:
                - ImageOverride -equals $(AgentName)

            variables:
              - group: NightlyBlobFuse
              - name: ROOT_DIR
                value: "/usr/pipeline/workv2"
              - name: WORK_DIR
                value: "/usr/pipeline/workv2/go/src/azure-storage-fuse"
              - name: skipComponentGovernanceDetection
                value: true
              - name: MOUNT_DIR
                value: "/usr/pipeline/workv2/blob_mnt"
              - name: TEMP_DIR
                value: "/usr/pipeline/workv2/temp"
              - name: BLOBFUSE2_CFG
                value: "/usr/pipeline/workv2/blobfuse2.yaml"
              - name: BLOBFUSE2_ADLS_CFG
                value: "/home/vsts/workv2/blobfuse2.adls.yaml"
              - name: GOPATH
                value: "/usr/pipeline/workv2/go"

            steps:
              # Go tool installer
              - task: GoTool@0
                inputs:
                  version: '1.20.5'
                displayName: "Install Go Version"

              - template: 'azure-pipeline-templates/distro-tests.yml'
                parameters:
                  working_dir: $(WORK_DIR)
                  root_dir: $(ROOT_DIR)
                  temp_dir: $(TEMP_DIR)
                  mount_dir: $(MOUNT_DIR)
                  config_path: $(BLOBFUSE2_CFG)
                  container: $(ContainerName)
                  blob_account_name: $(NIGHTLY_STO_BLOB_ACC_NAME)
                  blob_account_key: $(NIGHTLY_STO_BLOB_ACC_KEY)
                  adls_account_name: $(AZTEST_ADLS_ACC_NAME)
                  adls_account_key: $(AZTEST_ADLS_KEY)
                  distro_name: $(AgentName)
                  tags: $(tags)
                  fuselib: $(fuselib)
                  gopath: $(GOPATH)
                  installStep:
                    script: |
                      sudo tdnf install build-essential git fuse fuse-devel python36 -y
                    displayName: 'Install fuse'
                  verbose_log: ${{ parameters.verbose_log }}

        - ${{ if eq(parameters.msi_test, true) }}:
          # -----------------------------------------------------------
          # Ubuntu-20.04 MSI tests
          - job: Set_11
            timeoutInMinutes: 60
            strategy:
              matrix:
                Ubuntu-20-MSI:
                  DistroVer: "Ubn20_MSI"
                  AgentName: "blobfuse-ubuntu20"
                  Description: "Ubuntu 20 MSI Test"

            pool:
              name: "blobfuse-ubuntu-pool"
              demands:
                - ImageOverride -equals $(AgentName)

            variables:
              - group: NightlyBlobFuse
              - name: ROOT_DIR
                value: "/home/vsts/workv2/"
              - name: WORK_DIR
                value: "/home/vsts/workv2/go/src/azure-storage-fuse"
              - name: skipComponentGovernanceDetection
                value: true
              - name: MOUNT_DIR
                value: "/home/vsts/workv2/blob_mnt"
              - name: TEMP_DIR
                value: "/home/vsts/workv2/blobfuse2tmp"
              - name: BLOBFUSE2_CFG
                value: "/home/vsts/workv2//myblobv2.msi.yaml"
              - name: BLOBFUSE2_CFG_ADLS
                value: "/home/vsts/workv2/myblobv2.msi.adls.yaml"
              - name: GOPATH
                value: "/home/vsts/workv2/go"
              - name: containerName
                value: "test-cnt-ubn-20-msi"

            steps:
              - checkout: none

              # Print the agent info 
              - script: |
                  echo $(Description)
                  hostnamectl
                displayName: 'Print Agent Info'
              
              # Install libfuse
              - script: |
                  sudo apt-get install make cmake gcc g++ libfuse3-dev fuse3 -y -o Dpkg::Options::="--force-confnew"
                  sudo apt-get update --fix-missing -o Dpkg::Options::="--force-confnew"
                displayName: 'Install Fuse'

              # Prestart cleanup
              - script: |
                  sudo fusermount3 -u $(MOUNT_DIR)
                  sudo kill -9 `pidof blobfuse2`
                  sudo rm -rf $(ROOT_DIR)
                displayName: 'PreBuild Cleanup'

              # Create directory structure
              - script: |
                  sudo rm -rf $(ROOT_DIR)
                  sudo mkdir -p $(ROOT_DIR)
                  sudo chown -R `whoami` $(ROOT_DIR)
                  chmod 777 $(ROOT_DIR)
                  mkdir -p $(ROOT_DIR)/go/src
                displayName: 'Create Directory Structure'

              # Checkout the code  
              - script: |
                  git clone https://github.com/Azure/azure-storage-fuse
                displayName: 'Checkout Code'
                workingDirectory: $(ROOT_DIR)/go/src

              # Pull the branch
              - script: |
                  git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`
                displayName: 'Checkout Branch'
                workingDirectory: $(WORK_DIR)

              # Build the code
              - template: 'azure-pipeline-templates/build.yml'
                parameters:
                  working_directory: $(WORK_DIR)
                  root_dir: $(ROOT_DIR)
                  mount_dir: $(MOUNT_DIR)
                  temp_dir: $(TEMP_DIR)
                  gopath: $(GOPATH)
                  container: $(containerName)
                  skip_msi: "false"
              
              # BlockBlob MSI Test
              - script: |
                  cd $(WORK_DIR)
                  $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_msi.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG)
                displayName: "Create MSI Config File"
                env:
                  NIGHTLY_STO_BLOB_ACC_NAME: $(AZTEST_BLOCK_ACC_NAME) 
                  NIGHTLY_MSI_APP_ID: $(AZTEST_APP_ID)
                  ACCOUNT_TYPE: 'block'
                  ACCOUNT_ENDPOINT: 'https://$(AZTEST_BLOCK_ACC_NAME).blob.core.windows.net'
                  VERBOSE_LOG: ${{ parameters.verbose_log }}
                continueOnError: false

              - script:
                  cat $(BLOBFUSE2_CFG)
                displayName: "Print config file"

              # Mount the cntainer and run basic tests
              - template: 'azure-pipeline-templates/verify-auth.yml'
                parameters:
                  working_dir: $(WORK_DIR)
                  mount_dir: $(MOUNT_DIR)
                  temp_dir: $(TEMP_DIR)
                  idstring: 'BlockBlob MSI'
                  distro_name: $(AgentName)
                  mountStep:
                    script: |
                      $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG)
                    displayName: 'AuthVerify MSI: Mount Container'
                    continueOnError: false

              # ADLS MSI Test
              - script: |
                  cd $(WORK_DIR)
                  $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_msi.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG)
                displayName: "Create MSI Config File"
                env:
                  NIGHTLY_STO_BLOB_ACC_NAME: $(AZTEST_ADLS_ACC_NAME) 
                  NIGHTLY_MSI_APP_ID: $(AZTEST_APP_ID)
                  ACCOUNT_TYPE: 'adls'
                  ACCOUNT_ENDPOINT: 'https://$(AZTEST_ADLS_ACC_NAME).dfs.core.windows.net'
                  VERBOSE_LOG: ${{ parameters.verbose_log }}
                continueOnError: false

              - script:
                  cat $(BLOBFUSE2_CFG)
                displayName: "Print config file"

              # Mount the cntainer and run basic tests
              - template: 'azure-pipeline-templates/verify-auth.yml'
                parameters:
                  working_dir: $(WORK_DIR)
                  mount_dir: $(MOUNT_DIR)
                  temp_dir: $(TEMP_DIR)
                  idstring: 'ADLS MSI'
                  distro_name: $(AgentName)
                  mountStep:
                    script: |
                      $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG)
                    displayName: 'AuthVerify MSI: Mount Container'
                    continueOnError: false

              # Cleanup
              - template: 'azure-pipeline-templates/cleanup.yml'
                parameters:
                  working_dir: $(WORK_DIR)
                  mount_dir: $(MOUNT_DIR)
                  temp_dir: $(TEMP_DIR)

  - ${{ if eq(parameters.data_validation, true) }}:
    - stage: DataValidationBlob
      jobs:
        # Ubuntu Tests
        - job: Set_1
          timeoutInMinutes: 300
          strategy:
            matrix:
              Ubuntu-20:
                imageName: 'ubuntu-20.04'
                containerName: 'test-cnt-ubn-20'
                fuselib: 'libfuse-dev'
                tags: 'fuse2'
              Ubuntu-22:
                imageName: 'ubuntu-22.04'
                containerName: 'test-cnt-ubn-22'
                fuselib: 'libfuse3-dev'
                tags: 'fuse3'
          pool:
            vmImage: $(imageName)

          variables:
            - group: NightlyBlobFuse
            - name: ROOT_DIR
              value: "/usr/pipeline/workv2"
            - name: WORK_DIR
              value: "/usr/pipeline/workv2/go/src/azure-storage-fuse"
            - name: skipComponentGovernanceDetection
              value: true
            - name: MOUNT_DIR
              value: "/usr/pipeline/workv2/blob_mnt"
            - name: TEMP_DIR
              value: "/usr/pipeline/workv2/temp"
            - name: BLOBFUSE2_CFG
              value: "/usr/pipeline/workv2/blobfuse2.yaml"
            - name: GOPATH
              value: "/usr/pipeline/workv2/go"

          steps:
            - checkout: none

            - template: 'azure-pipeline-templates/setup.yml'
              parameters:
                tags: $(tags)
                installStep:
                  script: |
                    sudo apt-get update --fix-missing
                    sudo apt-get install $(fuselib) -y
                  displayName: 'Install fuse'

            - template: 'azure-pipeline-templates/e2e-tests-spcl.yml'
              parameters:
                conf_template: azure_key.yaml
                config_file: $(BLOBFUSE2_CFG)
                container: $(containerName)
                idstring: Block_Blob
                adls: false
                account_name: $(NIGHTLY_STO_BLOB_ACC_NAME)
                account_key: $(NIGHTLY_STO_BLOB_ACC_KEY)
                account_type: block
                account_endpoint: https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net
                distro_name: $(imageName)
                quick_test: false
                verbose_log: ${{ parameters.verbose_log }}
                clone: true
                stream_direct_test: false
                # TODO: These can be removed one day and replace all instances of ${{ parameters.temp_dir }} with $(TEMP_DIR) since it is a global variable
                temp_dir: $(TEMP_DIR)
                mount_dir: $(MOUNT_DIR)

  - ${{ if eq(parameters.data_validation, true) }}:
    - stage: DataValidationADLS
      jobs:
        # Ubuntu Tests
        - job: Set_1
          timeoutInMinutes: 300
          strategy:
            matrix:
              Ubuntu-20:
                imageName: 'ubuntu-20.04'
                containerName: 'test-cnt-ubn-20'
                fuselib: 'libfuse-dev'
                tags: 'fuse2'
              Ubuntu-22:
                imageName: 'ubuntu-22.04'
                containerName: 'test-cnt-ubn-22'
                fuselib: 'libfuse3-dev'
                tags: 'fuse3'
          pool:
            vmImage: $(imageName)

          variables:
            - group: NightlyBlobFuse
            - name: ROOT_DIR
              value: "/usr/pipeline/workv2"
            - name: WORK_DIR
              value: "/usr/pipeline/workv2/go/src/azure-storage-fuse"
            - name: skipComponentGovernanceDetection
              value: true
            - name: MOUNT_DIR
              value: "/usr/pipeline/workv2/blob_mnt"
            - name: TEMP_DIR
              value: "/usr/pipeline/workv2/temp"
            - name: BLOBFUSE2_CFG
              value: "/usr/pipeline/workv2/blobfuse2.yaml"
            - name: GOPATH
              value: "/usr/pipeline/workv2/go"

          steps:
            - checkout: none

            - template: 'azure-pipeline-templates/setup.yml'
              parameters:
                tags: $(tags)
                installStep:
                  script: |
                    sudo apt-get update --fix-missing
                    sudo apt-get install $(fuselib) -y
                  displayName: 'Install fuse'

            - template: 'azure-pipeline-templates/e2e-tests-spcl.yml'
              parameters:
                conf_template: azure_key.yaml
                config_file: $(BLOBFUSE2_CFG)
                container: $(containerName)
                idstring: ADLS
                adls: true
                account_name: $(AZTEST_ADLS_ACC_NAME)
                account_key: $(AZTEST_ADLS_KEY)
                account_type: adls
                account_endpoint: https://$(AZTEST_ADLS_ACC_NAME).dfs.core.windows.net
                distro_name: $(imageName)
                quick_test: false
                verbose_log: ${{ parameters.verbose_log }}
                clone: true
                stream_direct_test: false
                # TODO: These can be removed one day and replace all instances of ${{ parameters.temp_dir }} with $(TEMP_DIR) since it is a global variable
                temp_dir: $(TEMP_DIR)
                mount_dir: $(MOUNT_DIR)

  - ${{ if eq(parameters.data_validation, true) }}:
    - stage: DataValidationStreamFileHandle
      jobs:
        # Ubuntu Tests
        - job: Set_1
          timeoutInMinutes: 300
          strategy:
            matrix:
              Ubuntu-20:
                imageName: 'ubuntu-20.04'
                containerName: 'test-cnt-ubn-20'
                fuselib: 'libfuse-dev'
                tags: 'fuse2'
              Ubuntu-22:
                imageName: 'ubuntu-22.04'
                containerName: 'test-cnt-ubn-22'
                fuselib: 'libfuse3-dev'
                tags: 'fuse3'
          pool:
            vmImage: $(imageName)

          variables:
            - group: NightlyBlobFuse
            - name: ROOT_DIR
              value: "/usr/pipeline/workv2"
            - name: WORK_DIR
              value: "/usr/pipeline/workv2/go/src/azure-storage-fuse"
            - name: skipComponentGovernanceDetection
              value: true
            - name: MOUNT_DIR
              value: "/usr/pipeline/workv2/blob_mnt"
            - name: TEMP_DIR
              value: "/usr/pipeline/workv2/temp"
            - name: BLOBFUSE2_CFG
              value: "/usr/pipeline/workv2/blobfuse2.yaml"
            - name: GOPATH
              value: "/usr/pipeline/workv2/go"

          steps:
            - checkout: none

            - template: 'azure-pipeline-templates/setup.yml'
              parameters:
                tags: $(tags)
                installStep:
                  script: |
                    sudo apt-get update --fix-missing
                    sudo apt-get install $(fuselib) -y
                  displayName: 'Install fuse'

            - template: 'azure-pipeline-templates/e2e-tests-spcl.yml'
              parameters:
                conf_template: azure_stream.yaml
                config_file: $(BLOBFUSE2_CFG)
                container: $(containerName)
                idstring: Stream_File_Handle
                adls: false
                account_name: $(NIGHTLY_STO_BLOB_ACC_NAME)
                account_key: $(NIGHTLY_STO_BLOB_ACC_KEY)
                account_type: block
                account_endpoint: https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net
                distro_name: $(imageName)
                quick_test: false
                verbose_log: ${{ parameters.verbose_log }}
                clone: false
                stream_direct_test: false
                # TODO: These can be removed one day and replace all instances of ${{ parameters.temp_dir }} with $(TEMP_DIR) since it is a global variable
                temp_dir: $(TEMP_DIR)
                mount_dir: $(MOUNT_DIR)

  - ${{ if eq(parameters.data_validation, true) }}:
    - stage: DataValidationStreamFileHandleDirect
      jobs:
        # Ubuntu Tests
        - job: Set_1
          timeoutInMinutes: 300
          strategy:
            matrix:
              Ubuntu-20:
                imageName: 'ubuntu-20.04'
                containerName: 'test-cnt-ubn-20'
                fuselib: 'libfuse-dev'
                tags: 'fuse2'
              Ubuntu-22:
                imageName: 'ubuntu-22.04'
                containerName: 'test-cnt-ubn-22'
                fuselib: 'libfuse3-dev'
                tags: 'fuse3'
          pool:
            vmImage: $(imageName)

          variables:
            - group: NightlyBlobFuse
            - name: ROOT_DIR
              value: "/usr/pipeline/workv2"
            - name: WORK_DIR
              value: "/usr/pipeline/workv2/go/src/azure-storage-fuse"
            - name: skipComponentGovernanceDetection
              value: true
            - name: MOUNT_DIR
              value: "/usr/pipeline/workv2/blob_mnt"
            - name: TEMP_DIR
              value: "/usr/pipeline/workv2/temp"
            - name: BLOBFUSE2_CFG
              value: "/usr/pipeline/workv2/blobfuse2.yaml"
            - name: GOPATH
              value: "/usr/pipeline/workv2/go"

          steps:
            - checkout: none

            - template: 'azure-pipeline-templates/setup.yml'
              parameters:
                tags: $(tags)
                installStep:
                  script: |
                    sudo apt-get update --fix-missing
                    sudo apt-get install $(fuselib) -y
                  displayName: 'Install fuse'

            - template: 'azure-pipeline-templates/e2e-tests-spcl.yml'
              parameters:
                conf_template: azure_stream_direct.yaml
                config_file: $(BLOBFUSE2_CFG)
                container: $(containerName)
                idstring: Stream_File_Handle_Direct
                adls: false
                account_name: $(NIGHTLY_STO_BLOB_ACC_NAME)
                account_key: $(NIGHTLY_STO_BLOB_ACC_KEY)
                account_type: block
                account_endpoint: https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net
                distro_name: $(imageName)
                quick_test: false
                verbose_log: ${{ parameters.verbose_log }}
                clone: false
                stream_direct_test: true
                # TODO: These can be removed one day and replace all instances of ${{ parameters.temp_dir }} with $(TEMP_DIR) since it is a global variable
                temp_dir: $(TEMP_DIR)
                mount_dir: $(MOUNT_DIR)

  - ${{ if eq(parameters.data_validation, true) }}:
    - stage: DataValidationStreamFileName
      jobs:
        # Ubuntu Tests
        - job: Set_1
          timeoutInMinutes: 300
          strategy:
            matrix:
              Ubuntu-20:
                imageName: 'ubuntu-20.04'
                containerName: 'test-cnt-ubn-20'
                fuselib: 'libfuse-dev'
                tags: 'fuse2'
              Ubuntu-22:
                imageName: 'ubuntu-22.04'
                containerName: 'test-cnt-ubn-22'
                fuselib: 'libfuse3-dev'
                tags: 'fuse3'
          pool:
            vmImage: $(imageName)

          variables:
            - group: NightlyBlobFuse
            - name: ROOT_DIR
              value: "/usr/pipeline/workv2"
            - name: WORK_DIR
              value: "/usr/pipeline/workv2/go/src/azure-storage-fuse"
            - name: skipComponentGovernanceDetection
              value: true
            - name: MOUNT_DIR
              value: "/usr/pipeline/workv2/blob_mnt"
            - name: TEMP_DIR
              value: "/usr/pipeline/workv2/temp"
            - name: BLOBFUSE2_CFG
              value: "/usr/pipeline/workv2/blobfuse2.yaml"
            - name: GOPATH
              value: "/usr/pipeline/workv2/go"

          steps:
            - checkout: none

            - template: 'azure-pipeline-templates/setup.yml'
              parameters:
                tags: $(tags)
                installStep:
                  script: |
                    sudo apt-get update --fix-missing
                    sudo apt-get install $(fuselib) -y
                  displayName: 'Install fuse'

            - template: 'azure-pipeline-templates/e2e-tests-spcl.yml'
              parameters:
                conf_template: azure_stream_filename.yaml
                config_file: $(BLOBFUSE2_CFG)
                container: $(containerName)
                idstring: Stream_File_Name
                adls: false
                account_name: $(NIGHTLY_STO_BLOB_ACC_NAME)
                account_key: $(NIGHTLY_STO_BLOB_ACC_KEY)
                account_type: block
                account_endpoint: https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net
                distro_name: $(imageName)
                quick_test: false
                verbose_log: ${{ parameters.verbose_log }}
                clone: false
                stream_direct_test: false
                # TODO: These can be removed one day and replace all instances of ${{ parameters.temp_dir }} with $(TEMP_DIR) since it is a global variable
                temp_dir: $(TEMP_DIR)
                mount_dir: $(MOUNT_DIR)

  - ${{ if eq(parameters.data_validation, true) }}:
    - stage: DataValidationStreamFileNameDirect
      jobs:
        # Ubuntu Tests
        - job: Set_1
          timeoutInMinutes: 300
          strategy:
            matrix:
              Ubuntu-20:
                imageName: 'ubuntu-20.04'
                containerName: 'test-cnt-ubn-20'
                fuselib: 'libfuse-dev'
                tags: 'fuse2'
              Ubuntu-22:
                imageName: 'ubuntu-22.04'
                containerName: 'test-cnt-ubn-22'
                fuselib: 'libfuse3-dev'
                tags: 'fuse3'
          pool:
            vmImage: $(imageName)

          variables:
            - group: NightlyBlobFuse
            - name: ROOT_DIR
              value: "/usr/pipeline/workv2"
            - name: WORK_DIR
              value: "/usr/pipeline/workv2/go/src/azure-storage-fuse"
            - name: skipComponentGovernanceDetection
              value: true
            - name: MOUNT_DIR
              value: "/usr/pipeline/workv2/blob_mnt"
            - name: TEMP_DIR
              value: "/usr/pipeline/workv2/temp"
            - name: BLOBFUSE2_CFG
              value: "/usr/pipeline/workv2/blobfuse2.yaml"
            - name: GOPATH
              value: "/usr/pipeline/workv2/go"

          steps:
            - checkout: none

            - template: 'azure-pipeline-templates/setup.yml'
              parameters:
                tags: $(tags)
                installStep:
                  script: |
                    sudo apt-get update --fix-missing
                    sudo apt-get install $(fuselib) -y
                  displayName: 'Install fuse'

            - template: 'azure-pipeline-templates/e2e-tests-spcl.yml'
              parameters:
                conf_template: azure_stream_filename_direct.yaml
                config_file: $(BLOBFUSE2_CFG)
                container: $(containerName)
                idstring: Stream_File_Name_Direct
                adls: false
                account_name: $(NIGHTLY_STO_BLOB_ACC_NAME)
                account_key: $(NIGHTLY_STO_BLOB_ACC_KEY)
                account_type: block
                account_endpoint: https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net
                distro_name: $(imageName)
                quick_test: false
                verbose_log: ${{ parameters.verbose_log }}
                clone: false
                stream_direct_test: true
                # TODO: These can be removed one day and replace all instances of ${{ parameters.temp_dir }} with $(TEMP_DIR) since it is a global variable
                temp_dir: $(TEMP_DIR)
                mount_dir: $(MOUNT_DIR)

  - ${{ if eq(parameters.healthmon_test, true) }}:
    - stage: Healthmon
      jobs:
        - job: Set_1
          timeoutInMinutes: 300
          strategy:
            matrix:
              Ubuntu-22:
                imageName: 'ubuntu-22.04'
                containerName: 'test-cnt-ubn-22'
                adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22)
                fuselib: 'libfuse3-dev'
                tags: 'fuse3'
          pool:
            vmImage: $(imageName)

          variables:
            - group: NightlyBlobFuse
            - name: MOUNT_DIR
              value: '$(Pipeline.Workspace)/blob_mnt'
            - name: TEMP_DIR
              value: '$(Pipeline.Workspace)/blobfuse2_tmp'
            - name: BLOBFUSE2_CFG
              value: '$(Pipeline.Workspace)/blobfuse2.yaml'
            - name: GOPATH
              value: '$(Pipeline.Workspace)/go'
            - name: ROOT_DIR
              value: '$(System.DefaultWorkingDirectory)' 
            - name: WORK_DIR
              value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse'

          steps:
            - checkout: none

            - script: |
                sudo apt-get update --fix-missing -o Dpkg::Options::="--force-confnew"
                sudo apt-get install make cmake gcc g++ parallel $(fuselib) -y -o Dpkg::Options::="--force-confnew"
              displayName: 'Install libfuse'

            # Clone the repo
            - script: |
                git clone https://github.com/Azure/azure-storage-fuse
              displayName: 'Checkout Code'
              workingDirectory: $(ROOT_DIR)

            # Checkout the branch
            - script: |
                git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`
              displayName: 'Checkout Branch'
              workingDirectory: $(WORK_DIR)

            # Create directory structure
            - script: |
                sudo mkdir -p $(ROOT_DIR)
                sudo chown -R `whoami` $(ROOT_DIR)
                chmod 777 $(ROOT_DIR)
              displayName: 'Create Directory Structure'

            # -------------------------------------------------------
             # Alternative custom script for Self-Hosted agents to install Go-lang
            - task: ShellScript@2
              inputs:
                scriptPath: "$(WORK_DIR)/go_installer.sh"
                args: "$(ROOT_DIR)/ 1.20.5"
              displayName: "GoTool Custom Setup"

            # Downloading Go dependency packages
            - task: Go@0
              inputs:
                command: 'get'
                arguments: '-d'
                workingDirectory: $(WORK_DIR)
              displayName: "Go Get"

            # Building our package
            - task: Go@0
              inputs:
                command: 'build'
                arguments: "-o blobfuse2"
                workingDirectory: $(WORK_DIR)
              displayName: "Go Build Blobfuse2"

            # Building our package
            - task: Go@0
              inputs:
                command: 'build'
                arguments: "-o bfusemon ./tools/health-monitor/"
                workingDirectory: $(WORK_DIR)
              displayName: "Go Build bfusemon"

            - script: |
                cd $(WORK_DIR)
                $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key_hmon.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG)
              displayName: Create Key Config File
              env:
                ACCOUNT_TYPE: 'block'
                NIGHTLY_STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME)
                NIGHTLY_STO_ACC_KEY: $(NIGHTLY_STO_BLOB_ACC_KEY)
                ACCOUNT_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net'
                VERBOSE_LOG: false
                USE_HTTP: false
                HMON_OUTPUT: $(WORK_DIR)
              continueOnError: false

            - script: cat $(BLOBFUSE2_CFG)
              displayName: Print config file

            - script:
                mkdir -p $(MOUNT_DIR)
                mkdir -p $(TEMP_DIR)
              displayName: "Create directory structure"

            - template: 'azure-pipeline-templates/mount.yml'
              parameters:
                working_dir: $(WORK_DIR)
                mount_dir: $(MOUNT_DIR)
                temp_dir: $(TEMP_DIR)
                prefix: "Mount"
                mountStep:
                  script: |
                    sudo ln -s `pwd`/bfusemon /usr/local/bin/bfusemon
                    ls -l /usr/local/bin/bfusemon*
                    $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG)
                  displayName: 'Mount Blobfuse2'
                  workingDirectory: $(WORK_DIR)
                  timeoutInMinutes: 3
                  continueOnError: false

            - script: |
                touch $(MOUNT_DIR)/h1.txt
                echo "123456" > $(MOUNT_DIR)/h2.txt
                cp *.md $(MOUNT_DIR)/
                mkdir $(MOUNT_DIR)/A
                mv $(MOUNT_DIR)/h1.txt $(MOUNT_DIR)/h5.txt
                mv $(MOUNT_DIR)/h*.* $(MOUNT_DIR)/A
                rm -rf $(MOUNT_DIR)/*.md
                rm -rf $(MOUNT_DIR)/A
              displayName: "Test for health monitor"
              workingDirectory: $(WORK_DIR)

            - script: |
                sleep 20s
                cat $(WORK_DIR)/monitor_*.json
              displayName: "Print bfusemon output"
              workingDirectory: $(WORK_DIR)

            - script: |
                sudo ps -aux | grep bfusemon
              displayName: 'Print bfusemon Process info'
              env:
                mount_dir: $(MOUNT_DIR)

            - template: 'azure-pipeline-templates/cleanup.yml'
              parameters:
                working_dir: $(WORK_DIR)
                mount_dir: $(MOUNT_DIR)
                temp_dir: $(TEMP_DIR)