azure-storage-fuse/azure-pipeline-templates/verbose-tests.yml

parameters:
  - name: service
    type: string
  - name: account_type
    type: string
  - name: account_endpoint
    type: string 
  - name: spn_account_endpoint
    type: string 
  - name: adls
    type: boolean
  - name: account_name
    type: string
  - name: spn_account_name
    type: string
  - name: account_key
    type: string
  - name: account_sas
    type: string
  - name: client_id
    type: string
  - name: tenant_id
    type: string
  - name: client_secret
    type: string
  - name: container
    type: string
  - name: config
    type: string
  - name: working_dir
    type: string
  - name: mount_dir
    type: string
  - name: temp_dir
    type: string
  - name: stress_dir
    type: string
  - name: huge_container
    type: string
  - name: quick_stress
    type: boolean
  - name: test_key_credential
    type: boolean
  - name: test_sas_credential
    type: boolean
  - name: test_spn_credential
    type: boolean
  - name: test_stream
    type: boolean
  - name: stream_config
    type: string
  - name: stream_filename_config
    type: string
  - name: test_azurite
    type: boolean
    default: false
  - name: sas_credential_config
    type: string
  - name: spn_credential_config
    type: string
  - name: azurite_config
    type: string
    default: ""
  - name: distro_name
    type: string
  - name: quick_test
    type: boolean
    default: "true"
  - name: verbose_log
    type: boolean
  - name: tags
    type: string

#--------------------------------------- Setup: End to end tests with different Storage configurations ------------------------------------------
# Create key credential config file if we need to test it
steps:
  - script: |
      cd ${{ parameters.working_dir }}
      ${{ parameters.working_dir }}/blobfuse2 gen-test-config --config-file=azure_key.yaml --container-name=${{ parameters.container }} --temp-path=${{ parameters.temp_dir }} --output-file=${{ parameters.config }}
    displayName: Create Key Config File
    env:
      ACCOUNT_TYPE: ${{ parameters.account_type }}
      NIGHTLY_STO_ACC_NAME: ${{ parameters.account_name }}
      NIGHTLY_STO_ACC_KEY: ${{ parameters.account_key }}
      ACCOUNT_ENDPOINT: ${{ parameters.account_endpoint }}
      VERBOSE_LOG: ${{ parameters.verbose_log }}
      USE_HTTP: false
    condition: ${{ parameters.test_key_credential }}
    continueOnError: false
      
  - script: cat ${{ parameters.config }}
    displayName: Print config file
    condition: ${{ parameters.test_key_credential }}

# Stream e2e
  - script: |
      cd ${{ parameters.working_dir }}
      ${{ parameters.working_dir }}/blobfuse2 gen-test-config --config-file=azure_stream.yaml --container-name=${{ parameters.container }} --output-file=${{ parameters.stream_config }}
    displayName: Create Stream Config File
    env:
      ACCOUNT_TYPE: ${{ parameters.account_type }}
      NIGHTLY_STO_ACC_NAME: ${{ parameters.account_name }}
      NIGHTLY_STO_ACC_KEY: ${{ parameters.account_key }}
      ACCOUNT_ENDPOINT: ${{ parameters.account_endpoint }}
      VERBOSE_LOG: ${{ parameters.verbose_log }}
    condition: ${{ parameters.test_stream }}
    continueOnError: false
      
  - script: cat ${{ parameters.stream_config }}
    displayName: Print Stream config file with Handle Level Caching
    condition: ${{ parameters.test_stream }}

# Stream e2e filename level caching
  - script: |
      cd ${{ parameters.working_dir }}
      ${{ parameters.working_dir }}/blobfuse2 gen-test-config --config-file=azure_stream_filename.yaml --container-name=${{ parameters.container }} --output-file=${{ parameters.stream_filename_config }}
    displayName: Create Stream Config File
    env:
      ACCOUNT_TYPE: ${{ parameters.account_type }}
      NIGHTLY_STO_ACC_NAME: ${{ parameters.account_name }}
      NIGHTLY_STO_ACC_KEY: ${{ parameters.account_key }}
      ACCOUNT_ENDPOINT: ${{ parameters.account_endpoint }}
      VERBOSE_LOG: ${{ parameters.verbose_log }}
    condition: ${{ parameters.test_stream }}
    continueOnError: false
      
  - script: cat ${{ parameters.stream_filename_config }}
    displayName: Print Stream config file with Filename Caching
    condition: ${{ parameters.test_stream }}

# Create sas credential config file if we need to test it
  - script: |
      cd ${{ parameters.working_dir }}
      ${{ parameters.working_dir }}/blobfuse2 gen-test-config --config-file=azure_sas.yaml --container-name=${{ parameters.container }} --temp-path=${{ parameters.temp_dir }} --output-file=${{ parameters.sas_credential_config }}
    displayName: Create SAS Config File
    env:
      NIGHTLY_STO_BLOB_ACC_NAME: ${{ parameters.account_name }}
      NIGHTLY_STO_ACC_SAS: ${{ parameters.account_sas }}
      ACCOUNT_TYPE: ${{ parameters.account_type }}
      ACCOUNT_ENDPOINT: ${{ parameters.account_endpoint }}
      VERBOSE_LOG: ${{ parameters.verbose_log }}
    condition: ${{ parameters.test_sas_credential }}
    continueOnError: false
      
  - script: cat ${{ parameters.sas_credential_config }}
    displayName: Print SAS config file
    condition: ${{ parameters.test_sas_credential }}

# Create spn credential config file if we need to test it
  - script: |
      cd ${{ parameters.working_dir }}
      ${{ parameters.working_dir }}/blobfuse2 gen-test-config --config-file=azure_spn.yaml --container-name=${{ parameters.container }} --temp-path=${{ parameters.temp_dir }} --output-file=${{ parameters.spn_credential_config }}
    displayName: Create SPN Config File
    env:
      NIGHTLY_SPN_ACC_NAME: ${{ parameters.spn_account_name }}
      NIGHTLY_SPN_CLIENT_ID: ${{ parameters.client_id }}
      NIGHTLY_SPN_TENANT_ID: ${{ parameters.tenant_id }}
      NIGHTLY_SPN_CLIENT_SECRET: ${{ parameters.client_secret }}
      ACCOUNT_TYPE: ${{ parameters.account_type }}
      ACCOUNT_ENDPOINT: ${{ parameters.spn_account_endpoint }}
      VERBOSE_LOG: ${{ parameters.verbose_log }}
    condition: ${{ parameters.test_spn_credential }}
    continueOnError: false
      
  - script: cat ${{ parameters.spn_credential_config }}
    displayName: Print SPN config file
    condition: ${{ parameters.test_spn_credential }}

  # Create azurite config file if we need to test it
  - script: |
      cd ${{ parameters.working_dir }}
      ${{ parameters.working_dir }}/blobfuse2 gen-test-config --config-file=azure_key.yaml --container-name=${{ parameters.container }} --temp-path=${{ parameters.temp_dir }} --output-file=${{ parameters.azurite_config }}
    displayName: Create Key Config File
    # Note: It is fine to hardcode these credentials since they are publicly documented fake credentials for Azurite
    env:
      ACCOUNT_TYPE: ${{ parameters.account_type }}
      NIGHTLY_STO_ACC_NAME: 'devstoreaccount1'
      NIGHTLY_STO_ACC_KEY: 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=='
      ACCOUNT_ENDPOINT: 'http://127.0.0.1:10000/devstoreaccount1'
      USE_HTTP: 'true'
      VERBOSE_LOG: ${{ parameters.verbose_log }}
    condition: ${{ parameters.test_azurite }}
    continueOnError: false
      
  - script: cat ${{ parameters.azurite_config }}
    condition: ${{ parameters.test_azurite }}
    displayName: Print Azurite config file

#--------------------------------------- Tests: End to end tests with different Storage configurations ------------------------------------------

  - ${{ if eq(parameters.test_key_credential, true) }}:
    - template: e2e-tests.yml
      parameters:
        working_dir: ${{ parameters.working_dir }}
        mount_dir: ${{ parameters.mount_dir }}
        temp_dir: ${{ parameters.temp_dir }}
        adls: ${{ parameters.adls }}
        idstring: ${{ parameters.service }} with Key Credentials
        distro_name: ${{ parameters.distro_name }}
        quick_test: ${{ parameters.quick_test }}
        artifact_name: '${{ parameters.distro_name }}_${{ parameters.service }}_key.txt'
        verbose_log: ${{ parameters.verbose_log }}
        mountStep:
          script: >
            ${{ parameters.working_dir }}/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.config }}
            --default-working-dir=${{ parameters.working_dir }}
          displayName: 'E2E Test: Mount with Key Credential Configuration'
          timeoutInMinutes: 3
          continueOnError: false

  - ${{ if eq(parameters.test_stream, true) }}:
    - template: e2e-tests.yml
      parameters:
        working_dir: ${{ parameters.working_dir }}
        temp_dir: ${{ parameters.temp_dir }}
        mount_dir: ${{ parameters.mount_dir }}
        adls: ${{ parameters.adls }}
        idstring: ${{ parameters.service }} with Streaming
        distro_name: ${{ parameters.distro_name }}
        quick_test: ${{ parameters.quick_test }}
        artifact_name: '${{ parameters.distro_name }}_${{ parameters.service }}_stream.txt'
        verbose_log: ${{ parameters.verbose_log }}
        mountStep:
          script: >
            ${{ parameters.working_dir }}/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.stream_config }}
            --default-working-dir=${{ parameters.working_dir }}
          displayName: 'E2E Test: Mount with Stream Configuration'
          timeoutInMinutes: 3
          continueOnError: false

  - ${{ if eq(parameters.test_stream, true) }}:
    - template: e2e-tests.yml
      parameters:
        working_dir: ${{ parameters.working_dir }}
        temp_dir: ${{ parameters.temp_dir }}
        mount_dir: ${{ parameters.mount_dir }}
        adls: ${{ parameters.adls }}
        idstring: ${{ parameters.service }} with Streaming with filename
        distro_name: ${{ parameters.distro_name }}
        quick_test: ${{ parameters.quick_test }}
        artifact_name: '${{ parameters.distro_name }}_${{ parameters.service }}_stream_with_filename.txt'
        verbose_log: ${{ parameters.verbose_log }}
        mountStep:
          script: >
            ${{ parameters.working_dir }}/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.stream_filename_config }}
            --default-working-dir=${{ parameters.working_dir }}
          displayName: 'E2E Test: Mount with Filename Stream Configuration'
          timeoutInMinutes: 3
          continueOnError: false

  - ${{ if eq(parameters.test_sas_credential, true) }}:
    - template: e2e-tests.yml
      parameters:
        working_dir: ${{ parameters.working_dir }}
        mount_dir: ${{ parameters.mount_dir }}
        temp_dir: ${{ parameters.temp_dir }}
        adls: ${{ parameters.adls }}
        idstring: ${{ parameters.service }} with SAS Credentials
        distro_name: ${{ parameters.distro_name }}
        artifact_name: '${{ parameters.distro_name }}_${{ parameters.service }}_sas.txt'
        verbose_log: ${{ parameters.verbose_log }}
        mountStep:
          script: >
            ${{ parameters.working_dir }}/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.sas_credential_config }}
            --default-working-dir=${{ parameters.working_dir }}
          displayName: 'E2E Test: Mount with SAS Credential Configuration'
          timeoutInMinutes: 3
          continueOnError: false

  - ${{ if eq(parameters.test_spn_credential, true) }}:
    - template: e2e-tests.yml
      parameters:
        working_dir: ${{ parameters.working_dir }}
        mount_dir: ${{ parameters.mount_dir }}
        temp_dir: ${{ parameters.temp_dir }}
        adls: ${{ parameters.adls }}
        idstring: ${{ parameters.service }} with SPN Credentials
        distro_name: ${{ parameters.distro_name }}
        artifact_name: '${{ parameters.distro_name }}_${{ parameters.service }}_spn.txt'
        verbose_log: ${{ parameters.verbose_log }}
        mountStep:
          script: >
            ${{ parameters.working_dir }}/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.spn_credential_config }}
            --default-working-dir=${{ parameters.working_dir }}
          displayName: 'E2E Test: Mount with SPN Credential Configuration'
          timeoutInMinutes: 3
          continueOnError: false

  - ${{ if eq(parameters.test_azurite, true) }}:
    - bash: |
        sudo apt-get install azure-cli -y
        sudo npm install -g azurite
        sudo mkdir azurite
        sudo azurite --silent --location azurite --debug azurite\debug.log &
        az storage container create -n ${{ parameters.container }} --connection-string "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;"
      displayName: 'Install and Run Azurite'

    - template: e2e-tests.yml
      parameters: 
        working_dir: ${{ parameters.working_dir }}
        mount_dir: ${{ parameters.mount_dir }}
        temp_dir: ${{ parameters.temp_dir }}
        adls: ${{ parameters.adls }}
        idstring: ${{ parameters.service }} with Azurite
        distro_name: ${{ parameters.distro_name }}
        quick_test: ${{ parameters.quick_test }}
        artifact_name: '${{ parameters.distro_name }}_${{ parameters.service }}_azurite.txt'
        verbose_log: ${{ parameters.verbose_log }}
        mountStep:
          script: >
            ${{ parameters.working_dir }}/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.azurite_config }}
            --default-working-dir=${{ parameters.working_dir }}
          displayName: 'E2E Test: Mount with Azurite Configuration'
          timeoutInMinutes: 3
          continueOnError: false
      
#--------------------------------------- Tests: End to end tests with different File Cache configurations ------------------------------------------

  - template: e2e-tests-spcl.yml
    parameters:
      conf_template: azure_key_lfu.yaml
      config_file: ${{ parameters.config }}
      container: ${{ parameters.container }}
      temp_dir: ${{ parameters.temp_dir }}
      mount_dir: ${{ parameters.mount_dir }}
      adls: ${{ parameters.adls }}
      account_name: ${{ parameters.account_name }}
      account_key: ${{ parameters.account_key }}
      account_type: ${{ parameters.account_type }}
      account_endpoint: ${{ parameters.account_endpoint }}
      idstring: "${{ parameters.service }} LFU policy"
      distro_name: ${{ parameters.distro_name }}
      verbose_log: ${{ parameters.verbose_log }}

  - template: e2e-tests-spcl.yml
    parameters:
      conf_template: azure_key_lru_purge.yaml
      config_file: ${{ parameters.config }}
      container: ${{ parameters.container }}
      temp_dir: ${{ parameters.temp_dir }}
      mount_dir: ${{ parameters.mount_dir }}
      adls: ${{ parameters.adls }}
      account_name: ${{ parameters.account_name }}
      account_key: ${{ parameters.account_key }}
      account_type: ${{ parameters.account_type }}
      account_endpoint: ${{ parameters.account_endpoint }}
      idstring: "${{ parameters.service }} LRU policy no timeout"
      distro_name: ${{ parameters.distro_name }}
      verbose_log: ${{ parameters.verbose_log }}

  - template: e2e-tests-spcl.yml
    parameters:
      conf_template: azure_key_emptyfile.yaml
      config_file: ${{ parameters.config }}
      container: ${{ parameters.container }}
      temp_dir: ${{ parameters.temp_dir }}
      mount_dir: ${{ parameters.mount_dir }}
      adls: ${{ parameters.adls }}
      account_name: ${{ parameters.account_name }}
      account_key: ${{ parameters.account_key }}
      account_type: ${{ parameters.account_type }}
      account_endpoint: ${{ parameters.account_endpoint }}
      idstring: "${{ parameters.service }} LRU policy create empty"
      distro_name: ${{ parameters.distro_name }}
      quick_test: ${{ parameters.quick_test }}
      verbose_log: ${{ parameters.verbose_log }}


  - template: e2e-tests-spcl.yml
    parameters:
      conf_template: azure_key_directio.yaml
      config_file: ${{ parameters.config }}
      container: ${{ parameters.container }}
      temp_dir: ${{ parameters.temp_dir }}
      mount_dir: ${{ parameters.mount_dir }}
      adls: ${{ parameters.adls }}
      account_name: ${{ parameters.account_name }}
      account_key: ${{ parameters.account_key }}
      account_type: ${{ parameters.account_type }}
      account_endpoint: ${{ parameters.account_endpoint }}
      idstring: "${{ parameters.service }} Direct IO tests"
      distro_name: ${{ parameters.distro_name }}
      quick_test: ${{ parameters.quick_test }}
      verbose_log: ${{ parameters.verbose_log }}


  - template: e2e-tests-spcl.yml
    parameters:
      conf_template: azure_key_symlink.yaml
      config_file: ${{ parameters.config }}
      container: ${{ parameters.container }}
      temp_dir: ${{ parameters.temp_dir }}
      mount_dir: ${{ parameters.mount_dir }}
      adls: ${{ parameters.adls }}
      account_name: ${{ parameters.account_name }}
      account_key: ${{ parameters.account_key }}
      account_type: ${{ parameters.account_type }}
      account_endpoint: ${{ parameters.account_endpoint }}
      idstring: "${{ parameters.service }} Symlink config tests"
      distro_name: ${{ parameters.distro_name }}
      quick_test: ${{ parameters.quick_test }}
      verbose_log: ${{ parameters.verbose_log }}
      enable_symlink_adls: true


#--------------------------------------- Setup: End to end tests with different File Cache configurations ------------------------------------------
  - script: |
      cd ${{ parameters.working_dir }}
      ${{ parameters.working_dir }}/blobfuse2 gen-test-config --config-file=azure_key_huge.yaml --container-name=${{ parameters.container }} --temp-path=${{ parameters.temp_dir }} --output-file=${{ parameters.config }}
    displayName: Create Config File
    env:
      NIGHTLY_STO_ACC_NAME: ${{ parameters.account_name }}
      NIGHTLY_STO_ACC_KEY: ${{ parameters.account_key }}
      ACCOUNT_TYPE: ${{ parameters.account_type }}
      ACCOUNT_ENDPOINT: ${{ parameters.account_endpoint }}
      VERBOSE_LOG: ${{ parameters.verbose_log }}
    continueOnError: false

  - script: cat ${{ parameters.config }}
    displayName: Print config file

  - template: mount-test.yml
    parameters:
      working_dir: ${{ parameters.working_dir }}
      mount_dir: ${{ parameters.mount_dir }}
      temp_dir: ${{ parameters.temp_dir }}
      config: ${{ parameters.config }}
      idstring: ${{ parameters.service }} Mount Test
      tags: $(tags)

  - template: stress-test.yml
    parameters:
      working_dir: ${{ parameters.working_dir }}
      mount_dir: ${{ parameters.mount_dir }}
      temp_dir: ${{ parameters.temp_dir }}
      stress_dir: ${{ parameters.stress_dir }}
      idstring: ${{ parameters.service }} Stress Test
      quick: '${{ parameters.quick_stress }}'
      distro_name: ${{ parameters.distro_name }}
      mountStep:
        script: >
          ${{ parameters.working_dir }}/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.config }} --default-working-dir=${{ parameters.working_dir }}
        displayName: 'StressTest: Mount'
        timeoutInMinutes: 3
        continueOnError: false

  - script: |
      cd ${{ parameters.working_dir }}
      ${{ parameters.working_dir }}/blobfuse2 gen-test-config --config-file=azure_key_huge.yaml --container-name=${{ parameters.huge_container }} --temp-path=${{ parameters.temp_dir }} --output-file=${{ parameters.config }}
    displayName: Create Config File
    env:
      NIGHTLY_STO_ACC_NAME: ${{ parameters.account_name }}
      NIGHTLY_STO_ACC_KEY: ${{ parameters.account_key }}
      ACCOUNT_TYPE: ${{ parameters.account_type }}
      ACCOUNT_ENDPOINT: ${{ parameters.account_endpoint }}
      VERBOSE_LOG: ${{ parameters.verbose_log }}
    continueOnError: false

  - script: cat ${{ parameters.config }}
    displayName: Print config file

  - template: huge-list-test.yml
    parameters:
      working_dir: ${{ parameters.working_dir }}
      mount_dir: ${{ parameters.mount_dir }}
      temp_dir: ${{ parameters.temp_dir }}
      idstring: ${{ parameters.service }} Huge List Test
      distro_name: ${{ parameters.distro_name }}
      mountStep:
        script: >
          ${{ parameters.working_dir }}/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.config }}
          --default-working-dir=${{ parameters.working_dir }}
        displayName: 'HugeList: Mount'
        continueOnError: false