1974 строки
78 KiB
YAML
Executable File
1974 строки
78 KiB
YAML
Executable File
# Blobfuse2 Nightly Build Pipeline
|
|
|
|
schedules:
|
|
# Cron string < minute hour day-of-month month day-of-week>
|
|
# * means all, for example '*' in day of month means everyday
|
|
# Run only on main branch
|
|
# 'always' controls whether to run only if there is a change or not
|
|
# Run this pipeline every 15:00 time
|
|
- cron: '0 15 * * *'
|
|
displayName: 'Daily midnight Blobfuse2 build'
|
|
branches:
|
|
include:
|
|
- main
|
|
|
|
parameters:
|
|
- name: base_test
|
|
# Quick test or an exhaustive test
|
|
displayName: 'Basic test'
|
|
type: boolean
|
|
default: true
|
|
|
|
- name: exhaustive_test
|
|
# Quick test or an exhaustive test
|
|
displayName: 'Exhaustive test'
|
|
type: boolean
|
|
default: true
|
|
|
|
- name: proxy_test
|
|
# Proxy tests
|
|
displayName: 'Proxy test'
|
|
type: boolean
|
|
default: true
|
|
|
|
- name: msi_test
|
|
# MSI auth based test suites to be run or not
|
|
displayName: 'MSI test'
|
|
type: boolean
|
|
default: true
|
|
|
|
- name: healthmon_test
|
|
displayName: 'Healthmon test'
|
|
type: boolean
|
|
default: true
|
|
|
|
- name: quick_stress
|
|
displayName: 'Quick Stress'
|
|
type: boolean
|
|
default: true
|
|
|
|
- name: data_validation
|
|
displayName: 'Data Validation Test'
|
|
type: boolean
|
|
default: true
|
|
|
|
- name: block_cache_validation
|
|
displayName: 'Block Cache Validation Test'
|
|
type: boolean
|
|
default: true
|
|
|
|
- name: verbose_log
|
|
displayName: 'Verbose Log'
|
|
type: boolean
|
|
default: false
|
|
|
|
|
|
stages:
|
|
- ${{ if eq(parameters.base_test, true) }}:
|
|
- stage: BuildAndTest
|
|
jobs:
|
|
# Ubuntu Tests
|
|
- job: Set_1
|
|
timeoutInMinutes: 300
|
|
|
|
strategy:
|
|
matrix:
|
|
Ubuntu-20-BlockBlob:
|
|
imageName: 'ubuntu-20.04'
|
|
containerName: 'test-cnt-ubn-20'
|
|
adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_20)
|
|
fuselib: 'libfuse-dev'
|
|
tags: 'fuse2'
|
|
Ubuntu-22-BlockBlob:
|
|
imageName: 'ubuntu-22.04'
|
|
containerName: 'test-cnt-ubn-22'
|
|
adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22)
|
|
fuselib: 'libfuse3-dev'
|
|
tags: 'fuse3'
|
|
pool:
|
|
vmImage: $(imageName)
|
|
|
|
variables:
|
|
- group: NightlyBlobFuse
|
|
- name: MOUNT_DIR
|
|
value: '$(Pipeline.Workspace)/blob_mnt'
|
|
- name: TEMP_DIR
|
|
value: '$(Pipeline.Workspace)/blobfuse2_tmp'
|
|
- name: BLOBFUSE2_CFG
|
|
value: '$(Pipeline.Workspace)/blobfuse2.yaml'
|
|
- name: BLOBFUSE2_SAS_CFG
|
|
value: '$(Pipeline.Workspace)/blobfuse2_sas_config.yaml'
|
|
- name: BLOBFUSE2_SPN_CFG
|
|
value: '$(Pipeline.Workspace)/blobfuse2_spn_config.yaml'
|
|
- name: BLOBFUSE2_STREAM_CFG
|
|
value: '$(Pipeline.Workspace)/blobfuse2_stream.yaml'
|
|
- name: BLOBFUSE2_STREAM_FILENAME_CFG
|
|
value: '$(Pipeline.Workspace)/blobfuse2_stream_filename.yaml'
|
|
- name: BLOBFUSE2_ADLS_CFG
|
|
value: '$(Pipeline.Workspace)/blobfuse2.adls.yaml'
|
|
- name: BLOBFUSE2_GTEST_CFG
|
|
value: '$(Pipeline.Workspace)/connection.yaml'
|
|
- name: BLOBFUSE2_AZURITE_CFG
|
|
value: '$(Pipeline.Workspace)/blobfuse2_azurite_config.yaml'
|
|
- name: BLOBFUSE2_STRESS_DIR
|
|
value: '$(Pipeline.Workspace)/blobfuse2_stress'
|
|
- name: DECODE_PERCENTS
|
|
value: false
|
|
- name: GOPATH
|
|
value: '$(Pipeline.Workspace)/go'
|
|
- name: ROOT_DIR
|
|
value: '$(System.DefaultWorkingDirectory)'
|
|
- name: WORK_DIR
|
|
value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse'
|
|
|
|
steps:
|
|
- checkout: none
|
|
|
|
# Clone the repo
|
|
- script: |
|
|
git clone https://github.com/Azure/azure-storage-fuse
|
|
displayName: 'Checkout Code'
|
|
workingDirectory: $(ROOT_DIR)
|
|
|
|
# Checkout the branch
|
|
- script: |
|
|
git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`
|
|
displayName: 'Checkout Branch'
|
|
workingDirectory: $(WORK_DIR)
|
|
|
|
- script: |
|
|
sudo apt-get update --fix-missing
|
|
sudo apt-get install $(fuselib) -y
|
|
if [ $(tags) == "fuse2" ]; then
|
|
sudo apt-get install fuse -y
|
|
fi
|
|
displayName: 'Install libfuse'
|
|
|
|
# -------------------------------------------------------
|
|
# Pull and build the code
|
|
- template: 'azure-pipeline-templates/build.yml'
|
|
parameters:
|
|
working_directory: $(WORK_DIR)
|
|
root_dir: $(Pipeline.Workspace)
|
|
mount_dir: $(MOUNT_DIR)
|
|
temp_dir: $(TEMP_DIR)
|
|
gopath: $(GOPATH)
|
|
container: $(containerName)
|
|
tags: $(tags)
|
|
fuselib: $(fuselib)
|
|
|
|
# -------------------------------------------------------
|
|
- template: 'azure-pipeline-templates/invalid-command-tests.yml'
|
|
parameters:
|
|
working_dir: $(WORK_DIR)
|
|
mount_dir: $(MOUNT_DIR)
|
|
|
|
- ${{ if eq(parameters.exhaustive_test, true) }}:
|
|
- template: 'azure-pipeline-templates/verbose-tests.yml'
|
|
parameters:
|
|
service: 'BlockBlob'
|
|
account_type: 'block'
|
|
account_endpoint: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net'
|
|
adls: false
|
|
account_name: $(NIGHTLY_STO_BLOB_ACC_NAME)
|
|
account_key: $(NIGHTLY_STO_BLOB_ACC_KEY)
|
|
account_sas: $(NIGHTLY_STO_ACC_SAS)
|
|
spn_account_name: $(AZTEST_BLOCK_ACC_NAME)
|
|
spn_account_endpoint: 'https://$(AZTEST_BLOCK_ACC_NAME).blob.core.windows.net'
|
|
client_id: $(AZTEST_CLIENT)
|
|
tenant_id: $(AZTEST_TENANT)
|
|
client_secret: $(AZTEST_SECRET)
|
|
container: $(containerName)
|
|
config: $(BLOBFUSE2_CFG)
|
|
working_dir: $(WORK_DIR)
|
|
mount_dir: $(MOUNT_DIR)
|
|
temp_dir: $(TEMP_DIR)
|
|
stress_dir: $(BLOBFUSE2_STRESS_DIR)
|
|
huge_container: 'testcnt1'
|
|
quick_stress: ${{ parameters.quick_stress }}
|
|
test_key_credential: true
|
|
test_sas_credential: true
|
|
test_spn_credential: true
|
|
test_stream: true
|
|
test_azurite: true
|
|
stream_config: $(BLOBFUSE2_STREAM_CFG)
|
|
stream_filename_config: $(BLOBFUSE2_STREAM_FILENAME_CFG)
|
|
sas_credential_config: $(BLOBFUSE2_SAS_CFG)
|
|
spn_credential_config: $(BLOBFUSE2_SPN_CFG)
|
|
azurite_config: $(BLOBFUSE2_AZURITE_CFG)
|
|
distro_name: $(imageName)
|
|
verbose_log: ${{ parameters.verbose_log }}
|
|
tags: $(tags)
|
|
|
|
- template: azure-pipeline-templates/cleanup.yml
|
|
parameters:
|
|
working_dir: $(WORK_DIR)
|
|
mount_dir: $(MOUNT_DIR)
|
|
temp_dir: $(TEMP_DIR)
|
|
|
|
- job: Set_1_ARM
|
|
timeoutInMinutes: 300
|
|
|
|
strategy:
|
|
matrix:
|
|
Ubuntu-22-ARM64-BlockBlob:
|
|
imageName: 'blobfuse-ubn22-arm64'
|
|
containerName: 'test-cnt-ubn-22-arm64'
|
|
adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22_ARM)
|
|
fuselib: 'libfuse3-dev'
|
|
tags: 'fuse3'
|
|
|
|
pool:
|
|
name: "blobfuse-ubn-arm64-pool"
|
|
demands:
|
|
- ImageOverride -equals $(imageName)
|
|
|
|
variables:
|
|
- group: NightlyBlobFuse
|
|
- name: MOUNT_DIR
|
|
value: '$(Pipeline.Workspace)/blob_mnt'
|
|
- name: TEMP_DIR
|
|
value: '$(Pipeline.Workspace)/blobfuse2_tmp'
|
|
- name: BLOBFUSE2_CFG
|
|
value: '$(Pipeline.Workspace)/blobfuse2.yaml'
|
|
- name: BLOBFUSE2_SAS_CFG
|
|
value: '$(Pipeline.Workspace)/blobfuse2_sas_config.yaml'
|
|
- name: BLOBFUSE2_SPN_CFG
|
|
value: '$(Pipeline.Workspace)/blobfuse2_spn_config.yaml'
|
|
- name: BLOBFUSE2_STREAM_CFG
|
|
value: '$(Pipeline.Workspace)/blobfuse2_stream.yaml'
|
|
- name: BLOBFUSE2_STREAM_FILENAME_CFG
|
|
value: '$(Pipeline.Workspace)/blobfuse2_stream_filename.yaml'
|
|
- name: BLOBFUSE2_ADLS_CFG
|
|
value: '$(Pipeline.Workspace)/blobfuse2.adls.yaml'
|
|
- name: BLOBFUSE2_GTEST_CFG
|
|
value: '$(Pipeline.Workspace)/connection.yaml'
|
|
- name: BLOBFUSE2_AZURITE_CFG
|
|
value: '$(Pipeline.Workspace)/blobfuse2_azurite_config.yaml'
|
|
- name: BLOBFUSE2_STRESS_DIR
|
|
value: '$(Pipeline.Workspace)/blobfuse2_stress'
|
|
- name: DECODE_PERCENTS
|
|
value: false
|
|
- name: GOPATH
|
|
value: '$(Pipeline.Workspace)/go'
|
|
- name: ROOT_DIR
|
|
value: '$(System.DefaultWorkingDirectory)'
|
|
- name: WORK_DIR
|
|
value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse'
|
|
|
|
steps:
|
|
- checkout: none
|
|
|
|
- script: |
|
|
hostnamectl
|
|
displayName: 'Host info'
|
|
workingDirectory: $(ROOT_DIR)
|
|
|
|
# Clone the repo
|
|
- script: |
|
|
git clone https://github.com/Azure/azure-storage-fuse
|
|
displayName: 'Checkout Code'
|
|
workingDirectory: $(ROOT_DIR)
|
|
|
|
# Checkout the branch
|
|
- script: |
|
|
git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`
|
|
displayName: 'Checkout Branch'
|
|
workingDirectory: $(WORK_DIR)
|
|
|
|
- script: |
|
|
sudo apt-get update --fix-missing
|
|
sudo apt-get install $(tags) $(fuselib) gcc -y
|
|
displayName: 'Install fuse'
|
|
|
|
# -------------------------------------------------------
|
|
# Pull and build the code
|
|
- template: 'azure-pipeline-templates/build.yml'
|
|
parameters:
|
|
working_directory: $(WORK_DIR)
|
|
root_dir: $(Pipeline.Workspace)
|
|
mount_dir: $(MOUNT_DIR)
|
|
temp_dir: $(TEMP_DIR)
|
|
gopath: $(GOPATH)
|
|
container: $(containerName)
|
|
tags: $(tags)
|
|
fuselib: $(fuselib)
|
|
|
|
# -------------------------------------------------------
|
|
- template: 'azure-pipeline-templates/invalid-command-tests.yml'
|
|
parameters:
|
|
working_dir: $(WORK_DIR)
|
|
mount_dir: $(MOUNT_DIR)
|
|
|
|
- ${{ if eq(parameters.exhaustive_test, true) }}:
|
|
- template: 'azure-pipeline-templates/verbose-tests.yml'
|
|
parameters:
|
|
service: 'BlockBlob'
|
|
account_type: 'block'
|
|
account_endpoint: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net'
|
|
adls: false
|
|
account_name: $(NIGHTLY_STO_BLOB_ACC_NAME)
|
|
account_key: $(NIGHTLY_STO_BLOB_ACC_KEY)
|
|
account_sas: $(NIGHTLY_STO_ACC_SAS)
|
|
spn_account_name: $(AZTEST_BLOCK_ACC_NAME)
|
|
spn_account_endpoint: 'https://$(AZTEST_BLOCK_ACC_NAME).blob.core.windows.net'
|
|
client_id: $(AZTEST_CLIENT)
|
|
tenant_id: $(AZTEST_TENANT)
|
|
client_secret: $(AZTEST_SECRET)
|
|
container: $(containerName)
|
|
config: $(BLOBFUSE2_CFG)
|
|
working_dir: $(WORK_DIR)
|
|
mount_dir: $(MOUNT_DIR)
|
|
temp_dir: $(TEMP_DIR)
|
|
stress_dir: $(BLOBFUSE2_STRESS_DIR)
|
|
huge_container: 'testcnt1'
|
|
quick_stress: ${{ parameters.quick_stress }}
|
|
test_key_credential: true
|
|
test_sas_credential: true
|
|
test_spn_credential: true
|
|
test_stream: true
|
|
test_azurite: false
|
|
stream_config: $(BLOBFUSE2_STREAM_CFG)
|
|
stream_filename_config: $(BLOBFUSE2_STREAM_FILENAME_CFG)
|
|
sas_credential_config: $(BLOBFUSE2_SAS_CFG)
|
|
spn_credential_config: $(BLOBFUSE2_SPN_CFG)
|
|
azurite_config: $(BLOBFUSE2_AZURITE_CFG)
|
|
distro_name: $(imageName)
|
|
verbose_log: ${{ parameters.verbose_log }}
|
|
tags: $(tags)
|
|
|
|
- template: azure-pipeline-templates/cleanup.yml
|
|
parameters:
|
|
working_dir: $(WORK_DIR)
|
|
mount_dir: $(MOUNT_DIR)
|
|
temp_dir: $(TEMP_DIR)
|
|
|
|
# Ubuntu Tests
|
|
- job: Set_2
|
|
timeoutInMinutes: 300
|
|
|
|
strategy:
|
|
matrix:
|
|
Ubuntu-20-ADLS:
|
|
imageName: 'ubuntu-20.04'
|
|
containerName: 'test-cnt-ubn-20'
|
|
adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_20)
|
|
fuselib: 'libfuse-dev'
|
|
tags: 'fuse2'
|
|
Ubuntu-22-ADLS:
|
|
imageName: 'ubuntu-22.04'
|
|
containerName: 'test-cnt-ubn-22'
|
|
adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22)
|
|
fuselib: 'libfuse3-dev'
|
|
tags: 'fuse3'
|
|
pool:
|
|
vmImage: $(imageName)
|
|
|
|
variables:
|
|
- group: NightlyBlobFuse
|
|
- name: MOUNT_DIR
|
|
value: '$(Pipeline.Workspace)/blob_mnt'
|
|
- name: TEMP_DIR
|
|
value: '$(Pipeline.Workspace)/blobfuse2_tmp'
|
|
- name: BLOBFUSE2_CFG
|
|
value: '$(Pipeline.Workspace)/blobfuse2.yaml'
|
|
- name: BLOBFUSE2_SAS_CFG
|
|
value: '$(Pipeline.Workspace)/blobfuse2_sas_config.yaml'
|
|
- name: BLOBFUSE2_SPN_CFG
|
|
value: '$(Pipeline.Workspace)/blobfuse2_spn_config.yaml'
|
|
- name: BLOBFUSE2_STREAM_CFG
|
|
value: '$(Pipeline.Workspace)/blobfuse2_stream.yaml'
|
|
- name: BLOBFUSE2_STREAM_FILENAME_CFG
|
|
value: '$(Pipeline.Workspace)/blobfuse2_stream_filename.yaml'
|
|
- name: BLOBFUSE2_ADLS_CFG
|
|
value: '$(Pipeline.Workspace)/blobfuse2.adls.yaml'
|
|
- name: BLOBFUSE2_GTEST_CFG
|
|
value: '$(Pipeline.Workspace)/connection.yaml'
|
|
- name: BLOBFUSE2_AZURITE_CFG
|
|
value: '$(Pipeline.Workspace)/blobfuse2_azurite_config.yaml'
|
|
- name: BLOBFUSE2_STRESS_DIR
|
|
value: '$(Pipeline.Workspace)/blobfuse2_stress'
|
|
- name: DECODE_PERCENTS
|
|
value: false
|
|
- name: GOPATH
|
|
value: '$(Pipeline.Workspace)/go'
|
|
- name: ROOT_DIR
|
|
value: '$(System.DefaultWorkingDirectory)'
|
|
- name: WORK_DIR
|
|
value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse'
|
|
|
|
steps:
|
|
- checkout: none
|
|
|
|
# Clone the repo
|
|
- script: |
|
|
git clone https://github.com/Azure/azure-storage-fuse
|
|
displayName: 'Checkout Code'
|
|
workingDirectory: $(ROOT_DIR)
|
|
|
|
# Checkout the branch
|
|
- script: |
|
|
git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`
|
|
displayName: 'Checkout Branch'
|
|
workingDirectory: $(WORK_DIR)
|
|
|
|
- script: |
|
|
sudo apt-get update --fix-missing
|
|
sudo apt-get install $(fuselib) -y
|
|
if [ $(tags) == "fuse2" ]; then
|
|
sudo apt-get install fuse -y
|
|
fi
|
|
displayName: 'Install libfuse'
|
|
|
|
# -------------------------------------------------------
|
|
# Pull and build the code
|
|
- template: 'azure-pipeline-templates/build.yml'
|
|
parameters:
|
|
working_directory: $(WORK_DIR)
|
|
root_dir: $(Pipeline.Workspace)
|
|
mount_dir: $(MOUNT_DIR)
|
|
temp_dir: $(TEMP_DIR)
|
|
gopath: $(GOPATH)
|
|
container: $(containerName)
|
|
tags: $(tags)
|
|
fuselib: $(fuselib)
|
|
skip_ut: true # Skip UT because Block Blob set runs it
|
|
|
|
# -------------------------------------------------------
|
|
- ${{ if eq(parameters.exhaustive_test, true) }}:
|
|
- template: 'azure-pipeline-templates/verbose-tests.yml'
|
|
parameters:
|
|
service: 'ADLS'
|
|
account_type: 'adls'
|
|
account_endpoint: 'https://$(AZTEST_ADLS_ACC_NAME).dfs.core.windows.net'
|
|
adls: true
|
|
account_name: $(AZTEST_ADLS_ACC_NAME)
|
|
account_key: $(AZTEST_ADLS_KEY)
|
|
account_sas: $(adlsSas)
|
|
spn_account_name: $(AZTEST_ADLS_ACC_NAME)
|
|
spn_account_endpoint: 'https://$(AZTEST_ADLS_ACC_NAME).dfs.core.windows.net'
|
|
client_id: $(AZTEST_CLIENT)
|
|
tenant_id: $(AZTEST_TENANT)
|
|
client_secret: $(AZTEST_SECRET)
|
|
container: $(containerName)
|
|
config: $(BLOBFUSE2_ADLS_CFG)
|
|
working_dir: $(WORK_DIR)
|
|
mount_dir: $(MOUNT_DIR)
|
|
temp_dir: $(TEMP_DIR)
|
|
stress_dir: $(BLOBFUSE2_STRESS_DIR)
|
|
huge_container: 'testcnt'
|
|
quick_stress: ${{ parameters.quick_stress }}
|
|
test_key_credential: true
|
|
test_sas_credential: true
|
|
test_spn_credential: true
|
|
test_stream: true
|
|
test_azurite: false
|
|
stream_config: $(BLOBFUSE2_STREAM_CFG)
|
|
stream_filename_config: $(BLOBFUSE2_STREAM_FILENAME_CFG)
|
|
sas_credential_config: $(BLOBFUSE2_SAS_CFG)
|
|
spn_credential_config: $(BLOBFUSE2_SPN_CFG)
|
|
azurite_config: $(BLOBFUSE2_AZURITE_CFG)
|
|
distro_name: $(imageName)
|
|
verbose_log: ${{ parameters.verbose_log }}
|
|
tags: $(tags)
|
|
|
|
- template: azure-pipeline-templates/cleanup.yml
|
|
parameters:
|
|
working_dir: $(WORK_DIR)
|
|
mount_dir: $(MOUNT_DIR)
|
|
temp_dir: $(TEMP_DIR)
|
|
|
|
- job: Set_2_ARM
|
|
timeoutInMinutes: 300
|
|
|
|
strategy:
|
|
matrix:
|
|
Ubuntu-22-ARM64-ADLS:
|
|
imageName: 'blobfuse-ubn22-arm64'
|
|
containerName: 'test-cnt-ubn-22-arm64'
|
|
adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22_ARM)
|
|
fuselib: 'libfuse3-dev'
|
|
tags: 'fuse3'
|
|
pool:
|
|
name: "blobfuse-ubn-arm64-pool"
|
|
demands:
|
|
- ImageOverride -equals $(imageName)
|
|
|
|
variables:
|
|
- group: NightlyBlobFuse
|
|
- name: MOUNT_DIR
|
|
value: '$(Pipeline.Workspace)/blob_mnt'
|
|
- name: TEMP_DIR
|
|
value: '$(Pipeline.Workspace)/blobfuse2_tmp'
|
|
- name: BLOBFUSE2_CFG
|
|
value: '$(Pipeline.Workspace)/blobfuse2.yaml'
|
|
- name: BLOBFUSE2_SAS_CFG
|
|
value: '$(Pipeline.Workspace)/blobfuse2_sas_config.yaml'
|
|
- name: BLOBFUSE2_SPN_CFG
|
|
value: '$(Pipeline.Workspace)/blobfuse2_spn_config.yaml'
|
|
- name: BLOBFUSE2_STREAM_CFG
|
|
value: '$(Pipeline.Workspace)/blobfuse2_stream.yaml'
|
|
- name: BLOBFUSE2_STREAM_FILENAME_CFG
|
|
value: '$(Pipeline.Workspace)/blobfuse2_stream_filename.yaml'
|
|
- name: BLOBFUSE2_ADLS_CFG
|
|
value: '$(Pipeline.Workspace)/blobfuse2.adls.yaml'
|
|
- name: BLOBFUSE2_GTEST_CFG
|
|
value: '$(Pipeline.Workspace)/connection.yaml'
|
|
- name: BLOBFUSE2_AZURITE_CFG
|
|
value: '$(Pipeline.Workspace)/blobfuse2_azurite_config.yaml'
|
|
- name: BLOBFUSE2_STRESS_DIR
|
|
value: '$(Pipeline.Workspace)/blobfuse2_stress'
|
|
- name: DECODE_PERCENTS
|
|
value: false
|
|
- name: GOPATH
|
|
value: '$(Pipeline.Workspace)/go'
|
|
- name: ROOT_DIR
|
|
value: '$(System.DefaultWorkingDirectory)'
|
|
- name: WORK_DIR
|
|
value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse'
|
|
|
|
steps:
|
|
- checkout: none
|
|
|
|
# Clone the repo
|
|
- script: |
|
|
git clone https://github.com/Azure/azure-storage-fuse
|
|
displayName: 'Checkout Code'
|
|
workingDirectory: $(ROOT_DIR)
|
|
|
|
# Checkout the branch
|
|
- script: |
|
|
git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`
|
|
displayName: 'Checkout Branch'
|
|
workingDirectory: $(WORK_DIR)
|
|
|
|
- script: |
|
|
sudo apt-get update --fix-missing
|
|
sudo apt-get install $(fuselib) gcc -y
|
|
displayName: 'Install fuse'
|
|
|
|
# -------------------------------------------------------
|
|
# Pull and build the code
|
|
- template: 'azure-pipeline-templates/build.yml'
|
|
parameters:
|
|
working_directory: $(WORK_DIR)
|
|
root_dir: $(Pipeline.Workspace)
|
|
mount_dir: $(MOUNT_DIR)
|
|
temp_dir: $(TEMP_DIR)
|
|
gopath: $(GOPATH)
|
|
container: $(containerName)
|
|
tags: $(tags)
|
|
fuselib: $(fuselib)
|
|
skip_ut: true # Skip UT because Block Blob set runs it
|
|
|
|
# -------------------------------------------------------
|
|
- ${{ if eq(parameters.exhaustive_test, true) }}:
|
|
- template: 'azure-pipeline-templates/verbose-tests.yml'
|
|
parameters:
|
|
service: 'ADLS'
|
|
account_type: 'adls'
|
|
account_endpoint: 'https://$(AZTEST_ADLS_ACC_NAME).dfs.core.windows.net'
|
|
adls: true
|
|
account_name: $(AZTEST_ADLS_ACC_NAME)
|
|
account_key: $(AZTEST_ADLS_KEY)
|
|
account_sas: $(adlsSas)
|
|
spn_account_name: $(AZTEST_ADLS_ACC_NAME)
|
|
spn_account_endpoint: 'https://$(AZTEST_ADLS_ACC_NAME).dfs.core.windows.net'
|
|
client_id: $(AZTEST_CLIENT)
|
|
tenant_id: $(AZTEST_TENANT)
|
|
client_secret: $(AZTEST_SECRET)
|
|
container: $(containerName)
|
|
config: $(BLOBFUSE2_ADLS_CFG)
|
|
working_dir: $(WORK_DIR)
|
|
mount_dir: $(MOUNT_DIR)
|
|
temp_dir: $(TEMP_DIR)
|
|
stress_dir: $(BLOBFUSE2_STRESS_DIR)
|
|
huge_container: 'testcnt'
|
|
quick_stress: ${{ parameters.quick_stress }}
|
|
test_key_credential: true
|
|
test_sas_credential: false
|
|
test_spn_credential: true
|
|
test_stream: true
|
|
test_azurite: false
|
|
stream_config: $(BLOBFUSE2_STREAM_CFG)
|
|
stream_filename_config: $(BLOBFUSE2_STREAM_FILENAME_CFG)
|
|
sas_credential_config: $(BLOBFUSE2_SAS_CFG)
|
|
spn_credential_config: $(BLOBFUSE2_SPN_CFG)
|
|
azurite_config: $(BLOBFUSE2_AZURITE_CFG)
|
|
distro_name: $(imageName)
|
|
verbose_log: ${{ parameters.verbose_log }}
|
|
tags: $(tags)
|
|
|
|
- template: azure-pipeline-templates/cleanup.yml
|
|
parameters:
|
|
working_dir: $(WORK_DIR)
|
|
mount_dir: $(MOUNT_DIR)
|
|
temp_dir: $(TEMP_DIR)
|
|
|
|
|
|
- ${{ if eq(parameters.proxy_test, true) }}:
|
|
# -----------------------------------------------------------
|
|
# Ubuntu-20.04 Proxy tests
|
|
- job: Set_3
|
|
timeoutInMinutes: 300
|
|
strategy:
|
|
matrix:
|
|
Ubuntu-20-Proxy:
|
|
imageName: 'ubuntu-20.04'
|
|
containerName: 'test-cnt-ubn-18-proxy'
|
|
pool:
|
|
vmImage: $(imageName)
|
|
|
|
variables:
|
|
- group: NightlyBlobFuse
|
|
- name: MOUNT_DIR
|
|
value: '$(Pipeline.Workspace)/blob_mnt'
|
|
- name: TEMP_DIR
|
|
value: '$(Pipeline.Workspace)/blobfuse2_tmp'
|
|
- name: BLOBFUSE2_CFG
|
|
value: '$(Pipeline.Workspace)/blobfuse2_proxy.yaml'
|
|
- name: BLOBFUSE2_ADLS_CFG
|
|
value: '$(Pipeline.Workspace)/blobfuse2_proxy.adls.yaml'
|
|
- name: DECODE_PERCENTS
|
|
value: false
|
|
- name: GOPATH
|
|
value: '$(Pipeline.Workspace)/go'
|
|
- name: ROOT_DIR
|
|
value: '$(System.DefaultWorkingDirectory)'
|
|
- name: WORK_DIR
|
|
value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse'
|
|
|
|
steps:
|
|
- checkout: none
|
|
|
|
# Start Proxy
|
|
- script: |
|
|
sudo apt-get update --fix-missing
|
|
sudo apt remove mitmproxy
|
|
sudo apt-get install python3-setuptools
|
|
sudo apt install python3-pip
|
|
sudo pip3 install werkzeug==2.3.7
|
|
sudo pip3 install mitmproxy
|
|
mitmdump -w proxy_logs.txt &
|
|
displayName: 'Install & Start Proxy'
|
|
|
|
# Configure Proxy cert & env
|
|
- script: |
|
|
sudo mkdir /usr/local/share/ca-certificates/extra
|
|
sudo cp ~/.mitmproxy/mitmproxy-ca-cert.cer /usr/local/share/ca-certificates/extra/mitmproxy-ca-cert.crt
|
|
sudo update-ca-certificates
|
|
export https_proxy="127.0.0.1:8080"
|
|
export http_proxy="127.0.0.1:8080"
|
|
displayName: 'Configure Proxy'
|
|
|
|
# Clone the repo
|
|
- script: |
|
|
git clone https://github.com/Azure/azure-storage-fuse
|
|
displayName: 'Checkout Code'
|
|
workingDirectory: $(ROOT_DIR)
|
|
|
|
# Checkout the branch
|
|
- script: |
|
|
git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`
|
|
displayName: 'Checkout Branch'
|
|
workingDirectory: $(WORK_DIR)
|
|
|
|
- script: |
|
|
sudo apt-get update --fix-missing
|
|
sudo apt-get install libfuse3-dev
|
|
displayName: 'Install fuse3'
|
|
|
|
# -------------------------------------------------------
|
|
# Pull, build and unit test the code
|
|
- template: 'azure-pipeline-templates/build.yml'
|
|
parameters:
|
|
working_directory: $(WORK_DIR)
|
|
root_dir: $(Pipeline.Workspace)
|
|
mount_dir: $(MOUNT_DIR)
|
|
temp_dir: $(TEMP_DIR)
|
|
gopath: $(GOPATH)
|
|
container: $(containerName)
|
|
proxy_address: "127.0.0.1:8080"
|
|
skip_ut: true
|
|
|
|
# -------------------------------------------------------
|
|
- script: |
|
|
cd $(WORK_DIR)
|
|
$(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key_proxy.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG)
|
|
displayName: 'Create Config File'
|
|
env:
|
|
NIGHTLY_STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME)
|
|
NIGHTLY_STO_ACC_KEY: $(NIGHTLY_STO_BLOB_ACC_KEY)
|
|
ACCOUNT_TYPE: 'block'
|
|
ACCOUNT_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net'
|
|
VERBOSE_LOG: ${{ parameters.verbose_log }}
|
|
continueOnError: false
|
|
|
|
- script:
|
|
cat $(BLOBFUSE2_CFG)
|
|
displayName: "Print config file"
|
|
|
|
# --------------------------------------------------
|
|
# End to End tests
|
|
- template: 'azure-pipeline-templates/e2e-tests.yml'
|
|
parameters:
|
|
working_dir: $(WORK_DIR)
|
|
mount_dir: $(MOUNT_DIR)
|
|
temp_dir: $(TEMP_DIR)
|
|
idstring: 'BlockBlob with Proxy and Key Credentials'
|
|
distro_name: $(imageName)
|
|
adls: false
|
|
artifact_name: 'blockblob_proxy_key.txt'
|
|
verbose_log: ${{ parameters.verbose_log }}
|
|
mountStep:
|
|
script: |
|
|
$(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) --default-working-dir=$(WORK_DIR)
|
|
displayName: 'E2E TEST: Mount'
|
|
timeoutInMinutes: 3
|
|
continueOnError: false
|
|
|
|
# --------------------------------------------------
|
|
- script: |
|
|
cd $(WORK_DIR)
|
|
$(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key_proxy.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_ADLS_CFG)
|
|
displayName: 'Create ADLS Config File'
|
|
env:
|
|
NIGHTLY_STO_ACC_NAME: $(AZTEST_ADLS_ACC_NAME)
|
|
NIGHTLY_STO_ACC_KEY: $(AZTEST_ADLS_KEY)
|
|
ACCOUNT_TYPE: 'adls'
|
|
ACCOUNT_ENDPOINT: 'https://$(AZTEST_ADLS_ACC_NAME).dfs.core.windows.net'
|
|
VERBOSE_LOG: ${{ parameters.verbose_log }}
|
|
continueOnError: false
|
|
|
|
- script:
|
|
cat $(BLOBFUSE2_ADLS_CFG)
|
|
displayName: "Print ADLS config file"
|
|
|
|
- template: 'azure-pipeline-templates/e2e-tests.yml'
|
|
parameters:
|
|
working_dir: $(WORK_DIR)
|
|
mount_dir: $(MOUNT_DIR)
|
|
temp_dir: $(TEMP_DIR)
|
|
idstring: 'ADLS with Proxy and Key Credentials'
|
|
distro_name: $(imageName)
|
|
adls: true
|
|
artifact_name: 'adls_proxy_key.txt'
|
|
verbose_log: ${{ parameters.verbose_log }}
|
|
mountStep:
|
|
script: |
|
|
$(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_ADLS_CFG) --default-working-dir=$(WORK_DIR)
|
|
displayName: 'FeatureTest ADLS: Mount'
|
|
timeoutInMinutes: 3
|
|
continueOnError: false
|
|
# ------------------------------------------------------------
|
|
# Auth Tests
|
|
|
|
# Block SAS test
|
|
- script: |
|
|
cd $(WORK_DIR)
|
|
$(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_sas_proxy.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG)
|
|
displayName: "Create SAS Config File"
|
|
env:
|
|
NIGHTLY_STO_BLOB_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME)
|
|
NIGHTLY_STO_ACC_SAS: $(NIGHTLY_STO_ACC_SAS)
|
|
ACCOUNT_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net'
|
|
VERBOSE_LOG: ${{ parameters.verbose_log }}
|
|
continueOnError: false
|
|
|
|
- script:
|
|
cat $(BLOBFUSE2_CFG)
|
|
displayName: "Print SAS config file"
|
|
|
|
- template: 'azure-pipeline-templates/verify-auth.yml'
|
|
parameters:
|
|
working_dir: $(WORK_DIR)
|
|
mount_dir: $(MOUNT_DIR)
|
|
temp_dir: $(TEMP_DIR)
|
|
idstring: 'Block SAS'
|
|
distro_name: $(imageName)
|
|
mountStep:
|
|
script: |
|
|
$(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) --default-working-dir=$(WORK_DIR)
|
|
displayName: 'AuthVerify-SAS: Mount'
|
|
continueOnError: false
|
|
|
|
# Block SPN and proxy test
|
|
- script: |
|
|
cd $(WORK_DIR)
|
|
$(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_spn_proxy.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG)
|
|
displayName: "Create SPN Config File"
|
|
env:
|
|
NIGHTLY_SPN_ACC_NAME: $(AZTEST_BLOCK_ACC_NAME)
|
|
NIGHTLY_SPN_CLIENT_ID: $(AZTEST_CLIENT)
|
|
NIGHTLY_SPN_TENANT_ID: $(AZTEST_TENANT)
|
|
NIGHTLY_SPN_CLIENT_SECRET: $(AZTEST_SECRET)
|
|
ACCOUNT_ENDPOINT: 'https://$(AZTEST_BLOCK_ACC_NAME).blob.core.windows.net'
|
|
VERBOSE_LOG: ${{ parameters.verbose_log }}
|
|
continueOnError: false
|
|
|
|
- script:
|
|
cat $(BLOBFUSE2_CFG)
|
|
displayName: "Print SPN Config File"
|
|
|
|
- template: 'azure-pipeline-templates/verify-auth.yml'
|
|
parameters:
|
|
working_dir: $(WORK_DIR)
|
|
mount_dir: $(MOUNT_DIR)
|
|
temp_dir: $(TEMP_DIR)
|
|
idstring: 'Block SPN'
|
|
distro_name: $(imageName)
|
|
mountStep:
|
|
script: |
|
|
$(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) --default-working-dir=$(WORK_DIR)
|
|
displayName: 'AuthVerify-SPN: Mount'
|
|
continueOnError: false
|
|
|
|
# ------------------------------------------------------------
|
|
- template: 'azure-pipeline-templates/cleanup.yml'
|
|
parameters:
|
|
working_dir: $(WORK_DIR)
|
|
mount_dir: $(MOUNT_DIR)
|
|
temp_dir: $(TEMP_DIR)
|
|
- script: |
|
|
kill -9 $(pgrep mitmdump)
|
|
displayName: 'Kill Proxy'
|
|
|
|
- ${{ if eq(parameters.exhaustive_test, true) }}:
|
|
# RHEL Tests
|
|
- job: Set_4
|
|
timeoutInMinutes: 60
|
|
strategy:
|
|
matrix:
|
|
RHEL-7.5:
|
|
DistroVer: "RHEL-7.5"
|
|
Description: "Red Hat Enterprise Linux 7.5"
|
|
AgentName: "blobfuse-rhel7_5"
|
|
ContainerName: "test-cnt-rhel-75"
|
|
tags: 'fuse3'
|
|
RHEL-8.6:
|
|
DistroVer: "RHEL-8.6"
|
|
Description: "Red Hat Enterprise Linux 8.6"
|
|
AgentName: "blobfuse-rhel8_6"
|
|
containerName: "test-cnt-rhel-86"
|
|
tags: 'fuse3'
|
|
RHEL-9.0:
|
|
DistroVer: "RHEL-9.0"
|
|
Description: "Red Hat Enterprise Linux 9.0"
|
|
AgentName: "blobfuse-rhel9"
|
|
containerName: "test-cnt-rhel-9"
|
|
tags: 'fuse3'
|
|
pool:
|
|
name: "blobfuse-rhel-pool"
|
|
demands:
|
|
- ImageOverride -equals $(AgentName)
|
|
|
|
variables:
|
|
- group: NightlyBlobFuse
|
|
- name: ROOT_DIR
|
|
value: "/usr/pipeline/workv2"
|
|
- name: WORK_DIR
|
|
value: "/usr/pipeline/workv2/go/src/azure-storage-fuse"
|
|
- name: skipComponentGovernanceDetection
|
|
value: true
|
|
- name: MOUNT_DIR
|
|
value: "/usr/pipeline/workv2/blob_mnt"
|
|
- name: TEMP_DIR
|
|
value: "/usr/pipeline/workv2/temp"
|
|
- name: BLOBFUSE2_CFG
|
|
value: "/usr/pipeline/workv2/blobfuse2.yaml"
|
|
- name: BLOBFUSE2_ADLS_CFG
|
|
value: "/home/vsts/workv2/blobfuse2.adls.yaml"
|
|
- name: GOPATH
|
|
value: "/usr/pipeline/workv2/go"
|
|
|
|
steps:
|
|
- script: |
|
|
sudo touch /etc/yum.repos.d/centos.repo
|
|
sudo sh -c 'echo -e "[centos-extras]\nname=Centos extras - $basearch\nbaseurl=http://mirror.centos.org/centos/7/extras/x86_64\nenabled=1\ngpgcheck=1\ngpgkey=http://centos.org/keys/RPM-GPG-KEY-CentOS-7" > /etc/yum.repos.d/centos.repo'
|
|
condition: eq(variables['AgentName'], 'blobfuse-rhel7_5')
|
|
displayName: "Update OS mirrors"
|
|
|
|
- template: 'azure-pipeline-templates/distro-tests.yml'
|
|
parameters:
|
|
working_dir: $(WORK_DIR)
|
|
root_dir: $(ROOT_DIR)
|
|
temp_dir: $(TEMP_DIR)
|
|
mount_dir: $(MOUNT_DIR)
|
|
config_path: $(BLOBFUSE2_CFG)
|
|
container: $(ContainerName)
|
|
blob_account_name: $(NIGHTLY_STO_BLOB_ACC_NAME)
|
|
blob_account_key: $(NIGHTLY_STO_BLOB_ACC_KEY)
|
|
adls_account_name: $(AZTEST_ADLS_ACC_NAME)
|
|
adls_account_key: $(AZTEST_ADLS_KEY)
|
|
distro_name: $(AgentName)
|
|
gopath: $(GOPATH)
|
|
tags: $(tags)
|
|
installStep:
|
|
script: |
|
|
sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo
|
|
sudo yum update -y
|
|
sudo yum groupinstall "Development Tools" -y
|
|
if [ $(AgentName) == "blobfuse-rhel7_5" ]; then
|
|
sudo yum-config-manager --save --setopt=rhui-rhel-7-server-dotnet-rhui-rpms.skip_if_unavailable=true
|
|
sudo yum install git fuse fuse3-libs fuse3-devel fuse3 rh-python36 -y
|
|
elif [ $(AgentName) == "blobfuse-rhel9" ]; then
|
|
sudo yum install git fuse fuse3-libs fuse3-devel fuse3 python3 -y --nobest --allowerasing
|
|
else
|
|
sudo yum install git fuse fuse3-libs fuse3-devel fuse3 python36 -y --nobest --allowerasing
|
|
fi
|
|
displayName: 'Install fuse'
|
|
verbose_log: ${{ parameters.verbose_log }}
|
|
|
|
# Centos Tests
|
|
- job: Set_5
|
|
timeoutInMinutes: 60
|
|
strategy:
|
|
matrix:
|
|
CentOS-7.9:
|
|
DistroVer: "CentOS-7.9"
|
|
Description: "CentOS 7.9"
|
|
AgentName: "blobfuse-centos7"
|
|
ContainerName: "test-cnt-cent-7"
|
|
CentOS-8.5:
|
|
DistroVer: "CentOS-8.5"
|
|
Description: "CentOS 8.5"
|
|
AgentName: "blobfuse-centos8"
|
|
ContainerName: "test-cnt-cent-8"
|
|
|
|
pool:
|
|
name: "blobfuse-centos-pool"
|
|
demands:
|
|
- ImageOverride -equals $(AgentName)
|
|
|
|
variables:
|
|
- group: NightlyBlobFuse
|
|
- name: ROOT_DIR
|
|
value: "/usr/pipeline/workv2"
|
|
- name: WORK_DIR
|
|
value: "/usr/pipeline/workv2/go/src/azure-storage-fuse"
|
|
- name: skipComponentGovernanceDetection
|
|
value: true
|
|
- name: MOUNT_DIR
|
|
value: "/usr/pipeline/workv2/blob_mnt"
|
|
- name: TEMP_DIR
|
|
value: "/usr/pipeline/workv2/temp"
|
|
- name: BLOBFUSE2_CFG
|
|
value: "/usr/pipeline/workv2/blobfuse2.yaml"
|
|
- name: BLOBFUSE2_ADLS_CFG
|
|
value: "/home/vsts/workv2/blobfuse2.adls.yaml"
|
|
- name: GOPATH
|
|
value: "/usr/pipeline/workv2/go"
|
|
|
|
steps:
|
|
- script: |
|
|
sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-*
|
|
sudo sed -i 's|baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*
|
|
condition: eq(variables['AgentName'], 'blobfuse-centos8')
|
|
displayName: "Update OS mirrors"
|
|
|
|
- template: 'azure-pipeline-templates/distro-tests.yml'
|
|
parameters:
|
|
working_dir: $(WORK_DIR)
|
|
root_dir: $(ROOT_DIR)
|
|
temp_dir: $(TEMP_DIR)
|
|
mount_dir: $(MOUNT_DIR)
|
|
config_path: $(BLOBFUSE2_CFG)
|
|
container: $(ContainerName)
|
|
blob_account_name: $(NIGHTLY_STO_BLOB_ACC_NAME)
|
|
blob_account_key: $(NIGHTLY_STO_BLOB_ACC_KEY)
|
|
adls_account_name: $(AZTEST_ADLS_ACC_NAME)
|
|
adls_account_key: $(AZTEST_ADLS_KEY)
|
|
distro_name: $(AgentName)
|
|
gopath: $(GOPATH)
|
|
installStep:
|
|
script: |
|
|
sudo yum update -y --skip-broken
|
|
if [ $(AgentName) == "blobfuse-centos8" ]; then
|
|
sudo yum install gcc gcc-c++ make git fuse fuse3 fuse3-devel python36 -y --nobest --allowerasing
|
|
else
|
|
sudo yum install gcc gcc-c++ make git fuse3 fuse3-devel python36 -y
|
|
fi
|
|
displayName: 'Install fuse'
|
|
verbose_log: ${{ parameters.verbose_log }}
|
|
|
|
# Oracle Tests
|
|
- job: Set_6
|
|
timeoutInMinutes: 60
|
|
strategy:
|
|
matrix:
|
|
Oracle-8.1:
|
|
DistroVer: "Oracle-8.1"
|
|
Description: "Oracle Linux 8.1"
|
|
AgentName: "blobfuse-oracle81"
|
|
ContainerName: "test-cnt-ora-81"
|
|
|
|
pool:
|
|
name: "blobfuse-oracle-pool"
|
|
demands:
|
|
- ImageOverride -equals $(AgentName)
|
|
|
|
variables:
|
|
- group: NightlyBlobFuse
|
|
- name: ROOT_DIR
|
|
value: "/usr/pipeline/workv2"
|
|
- name: WORK_DIR
|
|
value: "/usr/pipeline/workv2/go/src/azure-storage-fuse"
|
|
- name: skipComponentGovernanceDetection
|
|
value: true
|
|
- name: MOUNT_DIR
|
|
value: "/usr/pipeline/workv2/blob_mnt"
|
|
- name: TEMP_DIR
|
|
value: "/usr/pipeline/workv2/temp"
|
|
- name: BLOBFUSE2_CFG
|
|
value: "/usr/pipeline/workv2/blobfuse2.yaml"
|
|
- name: BLOBFUSE2_ADLS_CFG
|
|
value: "/home/vsts/workv2/blobfuse2.adls.yaml"
|
|
- name: GOPATH
|
|
value: "/usr/pipeline/workv2/go"
|
|
|
|
steps:
|
|
- template: 'azure-pipeline-templates/distro-tests.yml'
|
|
parameters:
|
|
working_dir: $(WORK_DIR)
|
|
root_dir: $(ROOT_DIR)
|
|
temp_dir: $(TEMP_DIR)
|
|
mount_dir: $(MOUNT_DIR)
|
|
config_path: $(BLOBFUSE2_CFG)
|
|
container: $(ContainerName)
|
|
blob_account_name: $(NIGHTLY_STO_BLOB_ACC_NAME)
|
|
blob_account_key: $(NIGHTLY_STO_BLOB_ACC_KEY)
|
|
adls_account_name: $(AZTEST_ADLS_ACC_NAME)
|
|
adls_account_key: $(AZTEST_ADLS_KEY)
|
|
distro_name: $(AgentName)
|
|
gopath: $(GOPATH)
|
|
installStep:
|
|
script: |
|
|
sudo yum update -y
|
|
sudo yum install gcc gcc-c++ make git fuse fuse3 fuse3-devel python36 -y --nobest --allowerasing
|
|
displayName: 'Install fuse'
|
|
verbose_log: ${{ parameters.verbose_log }}
|
|
|
|
# SUSE Tests
|
|
- job: Set_9
|
|
timeoutInMinutes: 60
|
|
strategy:
|
|
matrix:
|
|
SUSE-15:
|
|
DistroVer: "SUSE-15"
|
|
Description: "SUSE Enterprise Linux 15"
|
|
AgentName: "blobfuse-suse15"
|
|
ContainerName: "test-cnt-suse-15"
|
|
|
|
pool:
|
|
name: "blobfuse-suse-pool"
|
|
demands:
|
|
- ImageOverride -equals $(AgentName)
|
|
|
|
variables:
|
|
- group: NightlyBlobFuse
|
|
- name: ROOT_DIR
|
|
value: "/usr/pipeline/workv2"
|
|
- name: WORK_DIR
|
|
value: "/usr/pipeline/workv2/go/src/azure-storage-fuse"
|
|
- name: skipComponentGovernanceDetection
|
|
value: true
|
|
- name: MOUNT_DIR
|
|
value: "/usr/pipeline/workv2/blob_mnt"
|
|
- name: TEMP_DIR
|
|
value: "/usr/pipeline/workv2/temp"
|
|
- name: BLOBFUSE2_CFG
|
|
value: "/usr/pipeline/workv2/blobfuse2.yaml"
|
|
- name: BLOBFUSE2_ADLS_CFG
|
|
value: "/home/vsts/workv2/blobfuse2.adls.yaml"
|
|
- name: GOPATH
|
|
value: "/usr/pipeline/workv2/go"
|
|
|
|
steps:
|
|
- template: 'azure-pipeline-templates/distro-tests.yml'
|
|
parameters:
|
|
working_dir: $(WORK_DIR)
|
|
root_dir: $(ROOT_DIR)
|
|
temp_dir: $(TEMP_DIR)
|
|
mount_dir: $(MOUNT_DIR)
|
|
config_path: $(BLOBFUSE2_CFG)
|
|
container: $(ContainerName)
|
|
blob_account_name: $(NIGHTLY_STO_BLOB_ACC_NAME)
|
|
blob_account_key: $(NIGHTLY_STO_BLOB_ACC_KEY)
|
|
adls_account_name: $(AZTEST_ADLS_ACC_NAME)
|
|
adls_account_key: $(AZTEST_ADLS_KEY)
|
|
distro_name: $(AgentName)
|
|
gopath: $(GOPATH)
|
|
installStep:
|
|
script: |
|
|
sudo zypper -n install git golang make cmake gcc gcc-c++ glibc-devel fuse
|
|
wget https://rpmfind.net/linux/opensuse/distribution/leap/15.4/repo/oss/x86_64/fuse3-3.10.5-150400.1.7.x86_64.rpm
|
|
sudo zypper -n --no-gpg-checks install fuse3-3.10.5-150400.1.7.x86_64.rpm
|
|
fusermount3 --v
|
|
wget https://rpmfind.net/linux/opensuse/distribution/leap/15.4/repo/oss/x86_64/fuse3-devel-3.10.5-150400.1.7.x86_64.rpm
|
|
sudo zypper -n --no-gpg-checks install fuse3-devel-3.10.5-150400.1.7.x86_64.rpm
|
|
displayName: 'Install fuse'
|
|
verbose_log: ${{ parameters.verbose_log }}
|
|
|
|
# Mariner Tests
|
|
- job: Set_10
|
|
timeoutInMinutes: 60
|
|
strategy:
|
|
matrix:
|
|
Mariner2:
|
|
DistroVer: "Mariner2"
|
|
Description: "CBL-Mariner2 Linux"
|
|
AgentName: "blobfuse-mariner2"
|
|
ContainerName: "test-cnt-mari-2"
|
|
fuselib: 'libfuse3-dev'
|
|
tags: 'fuse3'
|
|
pool:
|
|
name: "blobfuse-mariner-pool"
|
|
demands:
|
|
- ImageOverride -equals $(AgentName)
|
|
|
|
variables:
|
|
- group: NightlyBlobFuse
|
|
- name: ROOT_DIR
|
|
value: "/usr/pipeline/workv2"
|
|
- name: WORK_DIR
|
|
value: "/usr/pipeline/workv2/go/src/azure-storage-fuse"
|
|
- name: skipComponentGovernanceDetection
|
|
value: true
|
|
- name: MOUNT_DIR
|
|
value: "/usr/pipeline/workv2/blob_mnt"
|
|
- name: TEMP_DIR
|
|
value: "/usr/pipeline/workv2/temp"
|
|
- name: BLOBFUSE2_CFG
|
|
value: "/usr/pipeline/workv2/blobfuse2.yaml"
|
|
- name: BLOBFUSE2_ADLS_CFG
|
|
value: "/home/vsts/workv2/blobfuse2.adls.yaml"
|
|
- name: GOPATH
|
|
value: "/usr/pipeline/workv2/go"
|
|
|
|
steps:
|
|
- template: 'azure-pipeline-templates/distro-tests.yml'
|
|
parameters:
|
|
working_dir: $(WORK_DIR)
|
|
root_dir: $(ROOT_DIR)
|
|
temp_dir: $(TEMP_DIR)
|
|
mount_dir: $(MOUNT_DIR)
|
|
config_path: $(BLOBFUSE2_CFG)
|
|
container: $(ContainerName)
|
|
blob_account_name: $(NIGHTLY_STO_BLOB_ACC_NAME)
|
|
blob_account_key: $(NIGHTLY_STO_BLOB_ACC_KEY)
|
|
adls_account_name: $(AZTEST_ADLS_ACC_NAME)
|
|
adls_account_key: $(AZTEST_ADLS_KEY)
|
|
distro_name: $(AgentName)
|
|
tags: $(tags)
|
|
fuselib: $(fuselib)
|
|
gopath: $(GOPATH)
|
|
installStep:
|
|
script: |
|
|
sudo tdnf install gcc build-essential git fuse3 fuse3-devel python3 -y
|
|
displayName: 'Install fuse'
|
|
verbose_log: ${{ parameters.verbose_log }}
|
|
|
|
- ${{ if eq(parameters.msi_test, true) }}:
|
|
# -----------------------------------------------------------
|
|
# Ubuntu-20.04 MSI tests
|
|
- job: Set_11
|
|
timeoutInMinutes: 60
|
|
strategy:
|
|
matrix:
|
|
Ubuntu-20-MSI:
|
|
DistroVer: "Ubn20_MSI"
|
|
AgentName: "blobfuse-ubuntu20"
|
|
Description: "Ubuntu 20 MSI Test"
|
|
|
|
pool:
|
|
name: "blobfuse-ubuntu-pool"
|
|
demands:
|
|
- ImageOverride -equals $(AgentName)
|
|
|
|
variables:
|
|
- group: NightlyBlobFuse
|
|
- name: ROOT_DIR
|
|
value: "/home/vsts/workv2/"
|
|
- name: WORK_DIR
|
|
value: "/home/vsts/workv2/go/src/azure-storage-fuse"
|
|
- name: skipComponentGovernanceDetection
|
|
value: true
|
|
- name: MOUNT_DIR
|
|
value: "/home/vsts/workv2/blob_mnt"
|
|
- name: TEMP_DIR
|
|
value: "/home/vsts/workv2/blobfuse2tmp"
|
|
- name: BLOBFUSE2_CFG
|
|
value: "/home/vsts/workv2//myblobv2.msi.yaml"
|
|
- name: BLOBFUSE2_CFG_ADLS
|
|
value: "/home/vsts/workv2/myblobv2.msi.adls.yaml"
|
|
- name: GOPATH
|
|
value: "/home/vsts/workv2/go"
|
|
- name: containerName
|
|
value: "test-cnt-ubn-20-msi"
|
|
|
|
steps:
|
|
- checkout: none
|
|
|
|
# Print the agent info
|
|
- script: |
|
|
echo $(Description)
|
|
hostnamectl
|
|
displayName: 'Print Agent Info'
|
|
|
|
# Install libfuse
|
|
- script: |
|
|
sudo apt-get update --fix-missing -o Dpkg::Options::="--force-confnew"
|
|
sudo apt-get install make cmake gcc g++ libfuse3-dev fuse3 -y -o Dpkg::Options::="--force-confnew"
|
|
displayName: 'Install Fuse'
|
|
|
|
# Prestart cleanup
|
|
- script: |
|
|
sudo fusermount3 -u $(MOUNT_DIR)
|
|
sudo kill -9 `pidof blobfuse2`
|
|
sudo rm -rf $(ROOT_DIR)
|
|
displayName: 'PreBuild Cleanup'
|
|
|
|
# Create directory structure
|
|
- script: |
|
|
sudo rm -rf $(ROOT_DIR)
|
|
sudo mkdir -p $(ROOT_DIR)
|
|
sudo chown -R `whoami` $(ROOT_DIR)
|
|
chmod 777 $(ROOT_DIR)
|
|
mkdir -p $(ROOT_DIR)/go/src
|
|
displayName: 'Create Directory Structure'
|
|
|
|
# Checkout the code
|
|
- script: |
|
|
git clone https://github.com/Azure/azure-storage-fuse
|
|
displayName: 'Checkout Code'
|
|
workingDirectory: $(ROOT_DIR)/go/src
|
|
|
|
# Pull the branch
|
|
- script: |
|
|
git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`
|
|
displayName: 'Checkout Branch'
|
|
workingDirectory: $(WORK_DIR)
|
|
|
|
# Build the code
|
|
- template: 'azure-pipeline-templates/build.yml'
|
|
parameters:
|
|
working_directory: $(WORK_DIR)
|
|
root_dir: $(ROOT_DIR)
|
|
mount_dir: $(MOUNT_DIR)
|
|
temp_dir: $(TEMP_DIR)
|
|
gopath: $(GOPATH)
|
|
container: $(containerName)
|
|
skip_msi: "false"
|
|
|
|
# BlockBlob MSI Test
|
|
- script: |
|
|
cd $(WORK_DIR)
|
|
$(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_msi.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG)
|
|
displayName: "Create MSI Config File"
|
|
env:
|
|
NIGHTLY_STO_BLOB_ACC_NAME: $(AZTEST_BLOCK_ACC_NAME)
|
|
NIGHTLY_MSI_APP_ID: $(AZTEST_APP_ID)
|
|
ACCOUNT_TYPE: 'block'
|
|
ACCOUNT_ENDPOINT: 'https://$(AZTEST_BLOCK_ACC_NAME).blob.core.windows.net'
|
|
VERBOSE_LOG: ${{ parameters.verbose_log }}
|
|
continueOnError: false
|
|
|
|
- script:
|
|
cat $(BLOBFUSE2_CFG)
|
|
displayName: "Print config file"
|
|
|
|
# Mount the cntainer and run basic tests
|
|
- template: 'azure-pipeline-templates/verify-auth.yml'
|
|
parameters:
|
|
working_dir: $(WORK_DIR)
|
|
mount_dir: $(MOUNT_DIR)
|
|
temp_dir: $(TEMP_DIR)
|
|
idstring: 'BlockBlob MSI'
|
|
distro_name: $(AgentName)
|
|
mountStep:
|
|
script: |
|
|
$(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG)
|
|
displayName: 'AuthVerify MSI: Mount Container'
|
|
continueOnError: false
|
|
|
|
# ADLS MSI Test
|
|
- script: |
|
|
cd $(WORK_DIR)
|
|
$(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_msi.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG)
|
|
displayName: "Create MSI Config File"
|
|
env:
|
|
NIGHTLY_STO_BLOB_ACC_NAME: $(AZTEST_ADLS_ACC_NAME)
|
|
NIGHTLY_MSI_APP_ID: $(AZTEST_APP_ID)
|
|
ACCOUNT_TYPE: 'adls'
|
|
ACCOUNT_ENDPOINT: 'https://$(AZTEST_ADLS_ACC_NAME).dfs.core.windows.net'
|
|
VERBOSE_LOG: ${{ parameters.verbose_log }}
|
|
continueOnError: false
|
|
|
|
- script:
|
|
cat $(BLOBFUSE2_CFG)
|
|
displayName: "Print config file"
|
|
|
|
# Mount the cntainer and run basic tests
|
|
- template: 'azure-pipeline-templates/verify-auth.yml'
|
|
parameters:
|
|
working_dir: $(WORK_DIR)
|
|
mount_dir: $(MOUNT_DIR)
|
|
temp_dir: $(TEMP_DIR)
|
|
idstring: 'ADLS MSI'
|
|
distro_name: $(AgentName)
|
|
mountStep:
|
|
script: |
|
|
$(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG)
|
|
displayName: 'AuthVerify MSI: Mount Container'
|
|
continueOnError: false
|
|
|
|
# Cleanup
|
|
- template: 'azure-pipeline-templates/cleanup.yml'
|
|
parameters:
|
|
working_dir: $(WORK_DIR)
|
|
mount_dir: $(MOUNT_DIR)
|
|
temp_dir: $(TEMP_DIR)
|
|
|
|
|
|
- ${{ if eq(parameters.block_cache_validation, true) }}:
|
|
- stage: DataValidationBlockCache
|
|
jobs:
|
|
# Ubuntu Tests
|
|
- job: Set_1
|
|
timeoutInMinutes: 300
|
|
strategy:
|
|
matrix:
|
|
Ubuntu-22:
|
|
imageName: 'ubuntu-22.04'
|
|
containerName: 'test-cnt-ubn-22'
|
|
fuselib: 'libfuse3-dev'
|
|
tags: 'fuse3'
|
|
pool:
|
|
vmImage: $(imageName)
|
|
|
|
variables:
|
|
- group: NightlyBlobFuse
|
|
- name: ROOT_DIR
|
|
value: "/usr/pipeline/workv2"
|
|
- name: WORK_DIR
|
|
value: "/usr/pipeline/workv2/go/src/azure-storage-fuse"
|
|
- name: skipComponentGovernanceDetection
|
|
value: true
|
|
- name: MOUNT_DIR
|
|
value: "/usr/pipeline/workv2/blob_mnt"
|
|
- name: TEMP_DIR
|
|
value: "/usr/pipeline/workv2/temp"
|
|
- name: BLOBFUSE2_CFG
|
|
value: "/usr/pipeline/workv2/blobfuse2.yaml"
|
|
- name: GOPATH
|
|
value: "/usr/pipeline/workv2/go"
|
|
|
|
steps:
|
|
- checkout: none
|
|
|
|
- template: 'azure-pipeline-templates/setup.yml'
|
|
parameters:
|
|
tags: $(tags)
|
|
installStep:
|
|
script: |
|
|
sudo apt-get update --fix-missing
|
|
sudo apt-get install $(fuselib) -y
|
|
displayName: 'Install fuse'
|
|
|
|
- template: 'azure-pipeline-templates/e2e-tests-block-cache.yml'
|
|
parameters:
|
|
conf_template: azure_key.yaml
|
|
config_file: $(BLOBFUSE2_CFG)
|
|
container: $(containerName)
|
|
idstring: Block_Blob
|
|
adls: false
|
|
account_name: $(NIGHTLY_STO_BLOB_ACC_NAME)
|
|
account_key: $(NIGHTLY_STO_BLOB_ACC_KEY)
|
|
account_type: block
|
|
account_endpoint: https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net
|
|
distro_name: $(imageName)
|
|
quick_test: false
|
|
verbose_log: ${{ parameters.verbose_log }}
|
|
clone: true
|
|
stream_direct_test: false
|
|
# TODO: These can be removed one day and replace all instances of ${{ parameters.temp_dir }} with $(TEMP_DIR) since it is a global variable
|
|
temp_dir: $(TEMP_DIR)
|
|
mount_dir: $(MOUNT_DIR)
|
|
|
|
- ${{ if eq(parameters.data_validation, true) }}:
|
|
- stage: DataValidationBlob
|
|
jobs:
|
|
# Ubuntu Tests
|
|
- job: Set_1
|
|
timeoutInMinutes: 300
|
|
strategy:
|
|
matrix:
|
|
Ubuntu-20:
|
|
imageName: 'ubuntu-20.04'
|
|
containerName: 'test-cnt-ubn-20'
|
|
fuselib: 'libfuse-dev'
|
|
tags: 'fuse2'
|
|
Ubuntu-22:
|
|
imageName: 'ubuntu-22.04'
|
|
containerName: 'test-cnt-ubn-22'
|
|
fuselib: 'libfuse3-dev'
|
|
tags: 'fuse3'
|
|
pool:
|
|
vmImage: $(imageName)
|
|
|
|
variables:
|
|
- group: NightlyBlobFuse
|
|
- name: ROOT_DIR
|
|
value: "/usr/pipeline/workv2"
|
|
- name: WORK_DIR
|
|
value: "/usr/pipeline/workv2/go/src/azure-storage-fuse"
|
|
- name: skipComponentGovernanceDetection
|
|
value: true
|
|
- name: MOUNT_DIR
|
|
value: "/usr/pipeline/workv2/blob_mnt"
|
|
- name: TEMP_DIR
|
|
value: "/usr/pipeline/workv2/temp"
|
|
- name: BLOBFUSE2_CFG
|
|
value: "/usr/pipeline/workv2/blobfuse2.yaml"
|
|
- name: GOPATH
|
|
value: "/usr/pipeline/workv2/go"
|
|
|
|
steps:
|
|
- checkout: none
|
|
|
|
- template: 'azure-pipeline-templates/setup.yml'
|
|
parameters:
|
|
tags: $(tags)
|
|
installStep:
|
|
script: |
|
|
sudo apt-get update --fix-missing
|
|
sudo apt-get install $(fuselib) -y
|
|
displayName: 'Install fuse'
|
|
|
|
- template: 'azure-pipeline-templates/e2e-tests-spcl.yml'
|
|
parameters:
|
|
conf_template: azure_key.yaml
|
|
config_file: $(BLOBFUSE2_CFG)
|
|
container: $(containerName)
|
|
idstring: Block_Blob
|
|
adls: false
|
|
account_name: $(NIGHTLY_STO_BLOB_ACC_NAME)
|
|
account_key: $(NIGHTLY_STO_BLOB_ACC_KEY)
|
|
account_type: block
|
|
account_endpoint: https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net
|
|
distro_name: $(imageName)
|
|
quick_test: false
|
|
verbose_log: ${{ parameters.verbose_log }}
|
|
clone: true
|
|
stream_direct_test: false
|
|
# TODO: These can be removed one day and replace all instances of ${{ parameters.temp_dir }} with $(TEMP_DIR) since it is a global variable
|
|
temp_dir: $(TEMP_DIR)
|
|
mount_dir: $(MOUNT_DIR)
|
|
|
|
- ${{ if eq(parameters.data_validation, true) }}:
|
|
- stage: DataValidationADLS
|
|
jobs:
|
|
# Ubuntu Tests
|
|
- job: Set_1
|
|
timeoutInMinutes: 300
|
|
strategy:
|
|
matrix:
|
|
Ubuntu-20:
|
|
imageName: 'ubuntu-20.04'
|
|
containerName: 'test-cnt-ubn-20'
|
|
fuselib: 'libfuse-dev'
|
|
tags: 'fuse2'
|
|
Ubuntu-22:
|
|
imageName: 'ubuntu-22.04'
|
|
containerName: 'test-cnt-ubn-22'
|
|
fuselib: 'libfuse3-dev'
|
|
tags: 'fuse3'
|
|
pool:
|
|
vmImage: $(imageName)
|
|
|
|
variables:
|
|
- group: NightlyBlobFuse
|
|
- name: ROOT_DIR
|
|
value: "/usr/pipeline/workv2"
|
|
- name: WORK_DIR
|
|
value: "/usr/pipeline/workv2/go/src/azure-storage-fuse"
|
|
- name: skipComponentGovernanceDetection
|
|
value: true
|
|
- name: MOUNT_DIR
|
|
value: "/usr/pipeline/workv2/blob_mnt"
|
|
- name: TEMP_DIR
|
|
value: "/usr/pipeline/workv2/temp"
|
|
- name: BLOBFUSE2_CFG
|
|
value: "/usr/pipeline/workv2/blobfuse2.yaml"
|
|
- name: GOPATH
|
|
value: "/usr/pipeline/workv2/go"
|
|
|
|
steps:
|
|
- checkout: none
|
|
|
|
- template: 'azure-pipeline-templates/setup.yml'
|
|
parameters:
|
|
tags: $(tags)
|
|
installStep:
|
|
script: |
|
|
sudo apt-get update --fix-missing
|
|
sudo apt-get install $(fuselib) -y
|
|
displayName: 'Install fuse'
|
|
|
|
- template: 'azure-pipeline-templates/e2e-tests-spcl.yml'
|
|
parameters:
|
|
conf_template: azure_key.yaml
|
|
config_file: $(BLOBFUSE2_CFG)
|
|
container: $(containerName)
|
|
idstring: ADLS
|
|
adls: true
|
|
account_name: $(AZTEST_ADLS_ACC_NAME)
|
|
account_key: $(AZTEST_ADLS_KEY)
|
|
account_type: adls
|
|
account_endpoint: https://$(AZTEST_ADLS_ACC_NAME).dfs.core.windows.net
|
|
distro_name: $(imageName)
|
|
quick_test: false
|
|
verbose_log: ${{ parameters.verbose_log }}
|
|
clone: true
|
|
stream_direct_test: false
|
|
# TODO: These can be removed one day and replace all instances of ${{ parameters.temp_dir }} with $(TEMP_DIR) since it is a global variable
|
|
temp_dir: $(TEMP_DIR)
|
|
mount_dir: $(MOUNT_DIR)
|
|
|
|
- ${{ if eq(parameters.data_validation, true) }}:
|
|
- stage: DataValidationStreamFileHandle
|
|
jobs:
|
|
# Ubuntu Tests
|
|
- job: Set_1
|
|
timeoutInMinutes: 300
|
|
strategy:
|
|
matrix:
|
|
Ubuntu-20:
|
|
imageName: 'ubuntu-20.04'
|
|
containerName: 'test-cnt-ubn-20'
|
|
fuselib: 'libfuse-dev'
|
|
tags: 'fuse2'
|
|
Ubuntu-22:
|
|
imageName: 'ubuntu-22.04'
|
|
containerName: 'test-cnt-ubn-22'
|
|
fuselib: 'libfuse3-dev'
|
|
tags: 'fuse3'
|
|
pool:
|
|
vmImage: $(imageName)
|
|
|
|
variables:
|
|
- group: NightlyBlobFuse
|
|
- name: ROOT_DIR
|
|
value: "/usr/pipeline/workv2"
|
|
- name: WORK_DIR
|
|
value: "/usr/pipeline/workv2/go/src/azure-storage-fuse"
|
|
- name: skipComponentGovernanceDetection
|
|
value: true
|
|
- name: MOUNT_DIR
|
|
value: "/usr/pipeline/workv2/blob_mnt"
|
|
- name: TEMP_DIR
|
|
value: "/usr/pipeline/workv2/temp"
|
|
- name: BLOBFUSE2_CFG
|
|
value: "/usr/pipeline/workv2/blobfuse2.yaml"
|
|
- name: GOPATH
|
|
value: "/usr/pipeline/workv2/go"
|
|
|
|
steps:
|
|
- checkout: none
|
|
|
|
- template: 'azure-pipeline-templates/setup.yml'
|
|
parameters:
|
|
tags: $(tags)
|
|
installStep:
|
|
script: |
|
|
sudo apt-get update --fix-missing
|
|
sudo apt-get install $(fuselib) -y
|
|
displayName: 'Install fuse'
|
|
|
|
- template: 'azure-pipeline-templates/e2e-tests-spcl.yml'
|
|
parameters:
|
|
conf_template: azure_stream.yaml
|
|
config_file: $(BLOBFUSE2_CFG)
|
|
container: $(containerName)
|
|
idstring: Stream_File_Handle
|
|
adls: false
|
|
account_name: $(NIGHTLY_STO_BLOB_ACC_NAME)
|
|
account_key: $(NIGHTLY_STO_BLOB_ACC_KEY)
|
|
account_type: block
|
|
account_endpoint: https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net
|
|
distro_name: $(imageName)
|
|
quick_test: false
|
|
verbose_log: ${{ parameters.verbose_log }}
|
|
clone: false
|
|
stream_direct_test: false
|
|
# TODO: These can be removed one day and replace all instances of ${{ parameters.temp_dir }} with $(TEMP_DIR) since it is a global variable
|
|
temp_dir: $(TEMP_DIR)
|
|
mount_dir: $(MOUNT_DIR)
|
|
|
|
- ${{ if eq(parameters.data_validation, true) }}:
|
|
- stage: DataValidationStreamFileHandleDirect
|
|
jobs:
|
|
# Ubuntu Tests
|
|
- job: Set_1
|
|
timeoutInMinutes: 300
|
|
strategy:
|
|
matrix:
|
|
Ubuntu-20:
|
|
imageName: 'ubuntu-20.04'
|
|
containerName: 'test-cnt-ubn-20'
|
|
fuselib: 'libfuse-dev'
|
|
tags: 'fuse2'
|
|
Ubuntu-22:
|
|
imageName: 'ubuntu-22.04'
|
|
containerName: 'test-cnt-ubn-22'
|
|
fuselib: 'libfuse3-dev'
|
|
tags: 'fuse3'
|
|
pool:
|
|
vmImage: $(imageName)
|
|
|
|
variables:
|
|
- group: NightlyBlobFuse
|
|
- name: ROOT_DIR
|
|
value: "/usr/pipeline/workv2"
|
|
- name: WORK_DIR
|
|
value: "/usr/pipeline/workv2/go/src/azure-storage-fuse"
|
|
- name: skipComponentGovernanceDetection
|
|
value: true
|
|
- name: MOUNT_DIR
|
|
value: "/usr/pipeline/workv2/blob_mnt"
|
|
- name: TEMP_DIR
|
|
value: "/usr/pipeline/workv2/temp"
|
|
- name: BLOBFUSE2_CFG
|
|
value: "/usr/pipeline/workv2/blobfuse2.yaml"
|
|
- name: GOPATH
|
|
value: "/usr/pipeline/workv2/go"
|
|
|
|
steps:
|
|
- checkout: none
|
|
|
|
- template: 'azure-pipeline-templates/setup.yml'
|
|
parameters:
|
|
tags: $(tags)
|
|
installStep:
|
|
script: |
|
|
sudo apt-get update --fix-missing
|
|
sudo apt-get install $(fuselib) -y
|
|
displayName: 'Install fuse'
|
|
|
|
- template: 'azure-pipeline-templates/e2e-tests-spcl.yml'
|
|
parameters:
|
|
conf_template: azure_stream_direct.yaml
|
|
config_file: $(BLOBFUSE2_CFG)
|
|
container: $(containerName)
|
|
idstring: Stream_File_Handle_Direct
|
|
adls: false
|
|
account_name: $(NIGHTLY_STO_BLOB_ACC_NAME)
|
|
account_key: $(NIGHTLY_STO_BLOB_ACC_KEY)
|
|
account_type: block
|
|
account_endpoint: https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net
|
|
distro_name: $(imageName)
|
|
quick_test: false
|
|
verbose_log: ${{ parameters.verbose_log }}
|
|
clone: false
|
|
stream_direct_test: true
|
|
# TODO: These can be removed one day and replace all instances of ${{ parameters.temp_dir }} with $(TEMP_DIR) since it is a global variable
|
|
temp_dir: $(TEMP_DIR)
|
|
mount_dir: $(MOUNT_DIR)
|
|
|
|
- ${{ if eq(parameters.data_validation, true) }}:
|
|
- stage: DataValidationStreamFileName
|
|
jobs:
|
|
# Ubuntu Tests
|
|
- job: Set_1
|
|
timeoutInMinutes: 300
|
|
strategy:
|
|
matrix:
|
|
Ubuntu-20:
|
|
imageName: 'ubuntu-20.04'
|
|
containerName: 'test-cnt-ubn-20'
|
|
fuselib: 'libfuse-dev'
|
|
tags: 'fuse2'
|
|
Ubuntu-22:
|
|
imageName: 'ubuntu-22.04'
|
|
containerName: 'test-cnt-ubn-22'
|
|
fuselib: 'libfuse3-dev'
|
|
tags: 'fuse3'
|
|
pool:
|
|
vmImage: $(imageName)
|
|
|
|
variables:
|
|
- group: NightlyBlobFuse
|
|
- name: ROOT_DIR
|
|
value: "/usr/pipeline/workv2"
|
|
- name: WORK_DIR
|
|
value: "/usr/pipeline/workv2/go/src/azure-storage-fuse"
|
|
- name: skipComponentGovernanceDetection
|
|
value: true
|
|
- name: MOUNT_DIR
|
|
value: "/usr/pipeline/workv2/blob_mnt"
|
|
- name: TEMP_DIR
|
|
value: "/usr/pipeline/workv2/temp"
|
|
- name: BLOBFUSE2_CFG
|
|
value: "/usr/pipeline/workv2/blobfuse2.yaml"
|
|
- name: GOPATH
|
|
value: "/usr/pipeline/workv2/go"
|
|
|
|
steps:
|
|
- checkout: none
|
|
|
|
- template: 'azure-pipeline-templates/setup.yml'
|
|
parameters:
|
|
tags: $(tags)
|
|
installStep:
|
|
script: |
|
|
sudo apt-get update --fix-missing
|
|
sudo apt-get install $(fuselib) -y
|
|
displayName: 'Install fuse'
|
|
|
|
- template: 'azure-pipeline-templates/e2e-tests-spcl.yml'
|
|
parameters:
|
|
conf_template: azure_stream_filename.yaml
|
|
config_file: $(BLOBFUSE2_CFG)
|
|
container: $(containerName)
|
|
idstring: Stream_File_Name
|
|
adls: false
|
|
account_name: $(NIGHTLY_STO_BLOB_ACC_NAME)
|
|
account_key: $(NIGHTLY_STO_BLOB_ACC_KEY)
|
|
account_type: block
|
|
account_endpoint: https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net
|
|
distro_name: $(imageName)
|
|
quick_test: false
|
|
verbose_log: ${{ parameters.verbose_log }}
|
|
clone: false
|
|
stream_direct_test: false
|
|
# TODO: These can be removed one day and replace all instances of ${{ parameters.temp_dir }} with $(TEMP_DIR) since it is a global variable
|
|
temp_dir: $(TEMP_DIR)
|
|
mount_dir: $(MOUNT_DIR)
|
|
|
|
- ${{ if eq(parameters.data_validation, true) }}:
|
|
- stage: DataValidationStreamFileNameDirect
|
|
jobs:
|
|
# Ubuntu Tests
|
|
- job: Set_1
|
|
timeoutInMinutes: 300
|
|
strategy:
|
|
matrix:
|
|
Ubuntu-20:
|
|
imageName: 'ubuntu-20.04'
|
|
containerName: 'test-cnt-ubn-20'
|
|
fuselib: 'libfuse-dev'
|
|
tags: 'fuse2'
|
|
Ubuntu-22:
|
|
imageName: 'ubuntu-22.04'
|
|
containerName: 'test-cnt-ubn-22'
|
|
fuselib: 'libfuse3-dev'
|
|
tags: 'fuse3'
|
|
pool:
|
|
vmImage: $(imageName)
|
|
|
|
variables:
|
|
- group: NightlyBlobFuse
|
|
- name: ROOT_DIR
|
|
value: "/usr/pipeline/workv2"
|
|
- name: WORK_DIR
|
|
value: "/usr/pipeline/workv2/go/src/azure-storage-fuse"
|
|
- name: skipComponentGovernanceDetection
|
|
value: true
|
|
- name: MOUNT_DIR
|
|
value: "/usr/pipeline/workv2/blob_mnt"
|
|
- name: TEMP_DIR
|
|
value: "/usr/pipeline/workv2/temp"
|
|
- name: BLOBFUSE2_CFG
|
|
value: "/usr/pipeline/workv2/blobfuse2.yaml"
|
|
- name: GOPATH
|
|
value: "/usr/pipeline/workv2/go"
|
|
|
|
steps:
|
|
- checkout: none
|
|
|
|
- template: 'azure-pipeline-templates/setup.yml'
|
|
parameters:
|
|
tags: $(tags)
|
|
installStep:
|
|
script: |
|
|
sudo apt-get update --fix-missing
|
|
sudo apt-get install $(fuselib) -y
|
|
displayName: 'Install fuse'
|
|
|
|
- template: 'azure-pipeline-templates/e2e-tests-spcl.yml'
|
|
parameters:
|
|
conf_template: azure_stream_filename_direct.yaml
|
|
config_file: $(BLOBFUSE2_CFG)
|
|
container: $(containerName)
|
|
idstring: Stream_File_Name_Direct
|
|
adls: false
|
|
account_name: $(NIGHTLY_STO_BLOB_ACC_NAME)
|
|
account_key: $(NIGHTLY_STO_BLOB_ACC_KEY)
|
|
account_type: block
|
|
account_endpoint: https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net
|
|
distro_name: $(imageName)
|
|
quick_test: false
|
|
verbose_log: ${{ parameters.verbose_log }}
|
|
clone: false
|
|
stream_direct_test: true
|
|
# TODO: These can be removed one day and replace all instances of ${{ parameters.temp_dir }} with $(TEMP_DIR) since it is a global variable
|
|
temp_dir: $(TEMP_DIR)
|
|
mount_dir: $(MOUNT_DIR)
|
|
|
|
- ${{ if eq(parameters.healthmon_test, true) }}:
|
|
- stage: Healthmon
|
|
jobs:
|
|
- job: Set_1
|
|
timeoutInMinutes: 300
|
|
strategy:
|
|
matrix:
|
|
Ubuntu-22:
|
|
imageName: 'ubuntu-22.04'
|
|
containerName: 'test-cnt-ubn-22'
|
|
adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22)
|
|
fuselib: 'libfuse3-dev'
|
|
tags: 'fuse3'
|
|
pool:
|
|
vmImage: $(imageName)
|
|
|
|
variables:
|
|
- group: NightlyBlobFuse
|
|
- name: MOUNT_DIR
|
|
value: '$(Pipeline.Workspace)/blob_mnt'
|
|
- name: TEMP_DIR
|
|
value: '$(Pipeline.Workspace)/blobfuse2_tmp'
|
|
- name: BLOBFUSE2_CFG
|
|
value: '$(Pipeline.Workspace)/blobfuse2.yaml'
|
|
- name: GOPATH
|
|
value: '$(Pipeline.Workspace)/go'
|
|
- name: ROOT_DIR
|
|
value: '$(System.DefaultWorkingDirectory)'
|
|
- name: WORK_DIR
|
|
value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse'
|
|
|
|
steps:
|
|
- checkout: none
|
|
|
|
- script: |
|
|
sudo apt-get update --fix-missing -o Dpkg::Options::="--force-confnew"
|
|
sudo apt-get install make cmake gcc g++ parallel $(fuselib) -y -o Dpkg::Options::="--force-confnew"
|
|
displayName: 'Install libfuse'
|
|
|
|
# Clone the repo
|
|
- script: |
|
|
git clone https://github.com/Azure/azure-storage-fuse
|
|
displayName: 'Checkout Code'
|
|
workingDirectory: $(ROOT_DIR)
|
|
|
|
# Checkout the branch
|
|
- script: |
|
|
git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`
|
|
displayName: 'Checkout Branch'
|
|
workingDirectory: $(WORK_DIR)
|
|
|
|
# Create directory structure
|
|
- script: |
|
|
sudo mkdir -p $(ROOT_DIR)
|
|
sudo chown -R `whoami` $(ROOT_DIR)
|
|
chmod 777 $(ROOT_DIR)
|
|
displayName: 'Create Directory Structure'
|
|
|
|
# -------------------------------------------------------
|
|
# Alternative custom script for Self-Hosted agents to install Go-lang
|
|
- task: ShellScript@2
|
|
inputs:
|
|
scriptPath: "$(WORK_DIR)/go_installer.sh"
|
|
args: "$(ROOT_DIR)/"
|
|
displayName: "GoTool Custom Setup"
|
|
|
|
# Downloading Go dependency packages
|
|
- task: Go@0
|
|
inputs:
|
|
command: 'get'
|
|
arguments: '-d'
|
|
workingDirectory: $(WORK_DIR)
|
|
displayName: "Go Get"
|
|
|
|
# Building our package
|
|
- task: Go@0
|
|
inputs:
|
|
command: 'build'
|
|
arguments: "-o blobfuse2"
|
|
workingDirectory: $(WORK_DIR)
|
|
displayName: "Go Build Blobfuse2"
|
|
|
|
# Building our package
|
|
- task: Go@0
|
|
inputs:
|
|
command: 'build'
|
|
arguments: "-o bfusemon ./tools/health-monitor/"
|
|
workingDirectory: $(WORK_DIR)
|
|
displayName: "Go Build bfusemon"
|
|
|
|
- script: |
|
|
cd $(WORK_DIR)
|
|
$(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key_hmon.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG)
|
|
displayName: Create Key Config File
|
|
env:
|
|
ACCOUNT_TYPE: 'block'
|
|
NIGHTLY_STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME)
|
|
NIGHTLY_STO_ACC_KEY: $(NIGHTLY_STO_BLOB_ACC_KEY)
|
|
ACCOUNT_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net'
|
|
VERBOSE_LOG: false
|
|
USE_HTTP: false
|
|
HMON_OUTPUT: $(WORK_DIR)
|
|
continueOnError: false
|
|
|
|
- script: cat $(BLOBFUSE2_CFG)
|
|
displayName: Print config file
|
|
|
|
- script:
|
|
mkdir -p $(MOUNT_DIR)
|
|
mkdir -p $(TEMP_DIR)
|
|
displayName: "Create directory structure"
|
|
|
|
- template: 'azure-pipeline-templates/mount.yml'
|
|
parameters:
|
|
working_dir: $(WORK_DIR)
|
|
mount_dir: $(MOUNT_DIR)
|
|
temp_dir: $(TEMP_DIR)
|
|
prefix: "Mount"
|
|
mountStep:
|
|
script: |
|
|
sudo ln -s `pwd`/bfusemon /usr/local/bin/bfusemon
|
|
ls -l /usr/local/bin/bfusemon*
|
|
$(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG)
|
|
displayName: 'Mount Blobfuse2'
|
|
workingDirectory: $(WORK_DIR)
|
|
timeoutInMinutes: 3
|
|
continueOnError: false
|
|
|
|
- script: |
|
|
sudo ps -aux | grep bfusemon
|
|
displayName: 'Print bfusemon Process info'
|
|
env:
|
|
mount_dir: $(MOUNT_DIR)
|
|
|
|
- script: |
|
|
touch $(MOUNT_DIR)/h1.txt
|
|
echo "123456" > $(MOUNT_DIR)/h2.txt
|
|
cp *.md $(MOUNT_DIR)/
|
|
mkdir $(MOUNT_DIR)/A
|
|
mv $(MOUNT_DIR)/h1.txt $(MOUNT_DIR)/h5.txt
|
|
mv $(MOUNT_DIR)/h*.* $(MOUNT_DIR)/A
|
|
rm -rf $(MOUNT_DIR)/*.md
|
|
rm -rf $(MOUNT_DIR)/A
|
|
displayName: "Test for health monitor"
|
|
workingDirectory: $(WORK_DIR)
|
|
|
|
- script: |
|
|
sleep 20s
|
|
cat $(WORK_DIR)/monitor_*.json
|
|
displayName: "Print bfusemon output"
|
|
workingDirectory: $(WORK_DIR)
|
|
|
|
- template: 'azure-pipeline-templates/cleanup.yml'
|
|
parameters:
|
|
working_dir: $(WORK_DIR)
|
|
mount_dir: $(MOUNT_DIR)
|
|
temp_dir: $(TEMP_DIR) |