# Blobfuse2 Nightly Build Pipeline schedules: # Cron string < minute hour day-of-month month day-of-week> # * means all, for example '*' in day of month means everyday # Run only on main branch # 'always' controls whether to run only if there is a change or not # Run this pipeline every 15:00 time - cron: '0 15 * * *' displayName: 'Daily midnight Blobfuse2 build' branches: include: - main parameters: - name: base_test # Quick test or an exhaustive test displayName: 'Basic test' type: boolean default: true - name: exhaustive_test # Quick test or an exhaustive test displayName: 'Exhaustive test' type: boolean default: true - name: proxy_test # Proxy tests displayName: 'Proxy test' type: boolean default: true - name: msi_test # MSI auth based test suites to be run or not displayName: 'MSI test' type: boolean default: true - name: healthmon_test displayName: 'Healthmon test' type: boolean default: true - name: quick_stress displayName: 'Quick Stress' type: boolean default: true - name: data_validation displayName: 'Data Validation Test' type: boolean default: true - name: verbose_log displayName: 'Verbose Log' type: boolean default: false stages: - ${{ if eq(parameters.base_test, true) }}: - stage: BuildAndTest jobs: # Ubuntu Tests - job: Set_1 timeoutInMinutes: 300 strategy: matrix: Ubuntu-20: imageName: 'ubuntu-20.04' containerName: 'test-cnt-ubn-20' adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_20) fuselib: 'libfuse-dev' tags: 'fuse2' Ubuntu-22: imageName: 'ubuntu-22.04' containerName: 'test-cnt-ubn-22' adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22) fuselib: 'libfuse3-dev' tags: 'fuse3' pool: vmImage: $(imageName) variables: - group: NightlyBlobFuse - name: MOUNT_DIR value: '$(Pipeline.Workspace)/blob_mnt' - name: TEMP_DIR value: '$(Pipeline.Workspace)/blobfuse2_tmp' - name: BLOBFUSE2_CFG value: '$(Pipeline.Workspace)/blobfuse2.yaml' - name: BLOBFUSE2_SAS_CFG value: '$(Pipeline.Workspace)/blobfuse2_sas_config.yaml' - name: BLOBFUSE2_SPN_CFG value: '$(Pipeline.Workspace)/blobfuse2_spn_config.yaml' - name: BLOBFUSE2_STREAM_CFG value: '$(Pipeline.Workspace)/blobfuse2_stream.yaml' - name: BLOBFUSE2_ADLS_CFG value: '$(Pipeline.Workspace)/blobfuse2.adls.yaml' - name: BLOBFUSE2_GTEST_CFG value: '$(Pipeline.Workspace)/connection.yaml' - name: BLOBFUSE2_AZURITE_CFG value: '$(Pipeline.Workspace)/blobfuse2_azurite_config.yaml' - name: BLOBFUSE2_STRESS_DIR value: '$(Pipeline.Workspace)/blobfuse2_stress' - name: DECODE_PERCENTS value: false - name: GOPATH value: '$(Pipeline.Workspace)/go' - name: ROOT_DIR value: '$(System.DefaultWorkingDirectory)' - name: WORK_DIR value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' steps: - checkout: none # Clone the repo - script: | git clone https://github.com/Azure/azure-storage-fuse displayName: 'Checkout Code' workingDirectory: $(ROOT_DIR) # Checkout the branch - script: | git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` displayName: 'Checkout Branch' workingDirectory: $(WORK_DIR) - script: | sudo apt-get update --fix-missing sudo apt-get install $(fuselib) -y displayName: 'Install libfuse' # ------------------------------------------------------- # Pull and build the code - template: 'azure-pipeline-templates/build.yml' parameters: working_directory: $(WORK_DIR) root_dir: $(Pipeline.Workspace) mount_dir: $(MOUNT_DIR) temp_dir: $(TEMP_DIR) gopath: $(GOPATH) container: $(containerName) tags: $(tags) fuselib: $(fuselib) # ------------------------------------------------------- - template: 'azure-pipeline-templates/invalid-command-tests.yml' parameters: working_dir: $(WORK_DIR) mount_dir: $(MOUNT_DIR) - ${{ if eq(parameters.exhaustive_test, true) }}: - template: 'azure-pipeline-templates/verbose-tests.yml' parameters: service: 'BlockBlob' account_type: 'block' account_endpoint: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net' adls: false account_name: $(NIGHTLY_STO_BLOB_ACC_NAME) account_key: $(NIGHTLY_STO_BLOB_ACC_KEY) account_sas: $(NIGHTLY_STO_ACC_SAS) spn_account_name: $(AZTEST_BLOCK_ACC_NAME) spn_account_endpoint: 'https://$(AZTEST_BLOCK_ACC_NAME).blob.core.windows.net' client_id: $(AZTEST_CLIENT) tenant_id: $(AZTEST_TENANT) client_secret: $(AZTEST_SECRET) container: $(containerName) config: $(BLOBFUSE2_CFG) working_dir: $(WORK_DIR) mount_dir: $(MOUNT_DIR) temp_dir: $(TEMP_DIR) stress_dir: $(BLOBFUSE2_STRESS_DIR) huge_container: 'testcnt1' quick_stress: ${{ parameters.quick_stress }} test_key_credential: true test_sas_credential: true test_spn_credential: true test_stream: true test_azurite: true stream_config: $(BLOBFUSE2_STREAM_CFG) sas_credential_config: $(BLOBFUSE2_SAS_CFG) spn_credential_config: $(BLOBFUSE2_SPN_CFG) azurite_config: $(BLOBFUSE2_AZURITE_CFG) distro_name: $(imageName) verbose_log: ${{ parameters.verbose_log }} - template: 'azure-pipeline-templates/verbose-tests.yml' parameters: service: 'ADLS' account_type: 'adls' account_endpoint: 'https://$(AZTEST_ADLS_ACC_NAME).dfs.core.windows.net' adls: true account_name: $(AZTEST_ADLS_ACC_NAME) account_key: $(AZTEST_ADLS_KEY) account_sas: $(adlsSas) spn_account_name: $(AZTEST_ADLS_ACC_NAME) spn_account_endpoint: 'https://$(AZTEST_ADLS_ACC_NAME).dfs.core.windows.net' client_id: $(AZTEST_CLIENT) tenant_id: $(AZTEST_TENANT) client_secret: $(AZTEST_SECRET) container: $(containerName) config: $(BLOBFUSE2_ADLS_CFG) working_dir: $(WORK_DIR) mount_dir: $(MOUNT_DIR) temp_dir: $(TEMP_DIR) stress_dir: $(BLOBFUSE2_STRESS_DIR) huge_container: 'testcnt' quick_stress: ${{ parameters.quick_stress }} test_key_credential: true test_sas_credential: true test_spn_credential: true test_stream: true test_azurite: false stream_config: $(BLOBFUSE2_STREAM_CFG) sas_credential_config: $(BLOBFUSE2_SAS_CFG) spn_credential_config: $(BLOBFUSE2_SPN_CFG) azurite_config: $(BLOBFUSE2_AZURITE_CFG) distro_name: $(imageName) verbose_log: ${{ parameters.verbose_log }} - template: azure-pipeline-templates/cleanup.yml parameters: working_dir: $(WORK_DIR) mount_dir: $(MOUNT_DIR) temp_dir: $(TEMP_DIR) - ${{ if eq(parameters.proxy_test, true) }}: # ----------------------------------------------------------- # Ubuntu-20.04 Proxy tests - job: Set_2 timeoutInMinutes: 300 strategy: matrix: Ubuntu-20-Proxy: imageName: 'ubuntu-20.04' containerName: 'test-cnt-ubn-18-proxy' pool: vmImage: $(imageName) variables: - group: NightlyBlobFuse - name: MOUNT_DIR value: '$(Pipeline.Workspace)/blob_mnt' - name: TEMP_DIR value: '$(Pipeline.Workspace)/blobfuse2_tmp' - name: BLOBFUSE2_CFG value: '$(Pipeline.Workspace)/blobfuse2_proxy.yaml' - name: BLOBFUSE2_ADLS_CFG value: '$(Pipeline.Workspace)/blobfuse2_proxy.adls.yaml' - name: DECODE_PERCENTS value: false - name: GOPATH value: '$(Pipeline.Workspace)/go' - name: ROOT_DIR value: '$(System.DefaultWorkingDirectory)' - name: WORK_DIR value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' steps: - checkout: none # Start Proxy - script: | sudo apt-get update --fix-missing sudo apt remove mitmproxy sudo apt-get install python3-setuptools sudo apt install python3-pip sudo pip3 install mitmproxy mitmdump -w proxy_logs.txt & displayName: 'Install & Start Proxy' # Configure Proxy cert & env - script: | sudo mkdir /usr/local/share/ca-certificates/extra sudo cp ~/.mitmproxy/mitmproxy-ca-cert.cer /usr/local/share/ca-certificates/extra/mitmproxy-ca-cert.crt sudo update-ca-certificates export https_proxy="127.0.0.1:8080" export http_proxy="127.0.0.1:8080" displayName: 'Configure Proxy' # Clone the repo - script: | git clone https://github.com/Azure/azure-storage-fuse displayName: 'Checkout Code' workingDirectory: $(ROOT_DIR) # Checkout the branch - script: | git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` displayName: 'Checkout Branch' workingDirectory: $(WORK_DIR) - script: | sudo apt-get update --fix-missing sudo apt-get install libfuse3-dev displayName: 'Install fuse3' # ------------------------------------------------------- # Pull, build and unit test the code - template: 'azure-pipeline-templates/build.yml' parameters: working_directory: $(WORK_DIR) root_dir: $(Pipeline.Workspace) mount_dir: $(MOUNT_DIR) temp_dir: $(TEMP_DIR) gopath: $(GOPATH) container: $(containerName) proxy_address: "127.0.0.1:8080" skip_ut: true # ------------------------------------------------------- - script: | cd $(WORK_DIR) $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key_proxy.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG) displayName: 'Create Config File' env: NIGHTLY_STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME) NIGHTLY_STO_ACC_KEY: $(NIGHTLY_STO_BLOB_ACC_KEY) ACCOUNT_TYPE: 'block' ACCOUNT_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net' VERBOSE_LOG: ${{ parameters.verbose_log }} continueOnError: false - script: cat $(BLOBFUSE2_CFG) displayName: "Print config file" # -------------------------------------------------- # End to End tests - template: 'azure-pipeline-templates/e2e-tests.yml' parameters: working_dir: $(WORK_DIR) mount_dir: $(MOUNT_DIR) temp_dir: $(TEMP_DIR) idstring: 'BlockBlob with Proxy and Key Credentials' distro_name: $(imageName) adls: false artifact_name: 'blockblob_proxy_key.txt' verbose_log: ${{ parameters.verbose_log }} mountStep: script: | $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) --default-working-dir=$(WORK_DIR) displayName: 'E2E TEST: Mount' timeoutInMinutes: 3 continueOnError: false # -------------------------------------------------- - script: | cd $(WORK_DIR) $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key_proxy.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_ADLS_CFG) displayName: 'Create ADLS Config File' env: NIGHTLY_STO_ACC_NAME: $(AZTEST_ADLS_ACC_NAME) NIGHTLY_STO_ACC_KEY: $(AZTEST_ADLS_KEY) ACCOUNT_TYPE: 'adls' ACCOUNT_ENDPOINT: 'https://$(AZTEST_ADLS_ACC_NAME).dfs.core.windows.net' VERBOSE_LOG: ${{ parameters.verbose_log }} continueOnError: false - script: cat $(BLOBFUSE2_ADLS_CFG) displayName: "Print ADLS config file" - template: 'azure-pipeline-templates/e2e-tests.yml' parameters: working_dir: $(WORK_DIR) mount_dir: $(MOUNT_DIR) temp_dir: $(TEMP_DIR) idstring: 'ADLS with Proxy and Key Credentials' distro_name: $(imageName) adls: true artifact_name: 'adls_proxy_key.txt' verbose_log: ${{ parameters.verbose_log }} mountStep: script: | $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_ADLS_CFG) --default-working-dir=$(WORK_DIR) displayName: 'FeatureTest ADLS: Mount' timeoutInMinutes: 3 continueOnError: false # ------------------------------------------------------------ # Auth Tests # Block SAS test - script: | cd $(WORK_DIR) $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_sas_proxy.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG) displayName: "Create SAS Config File" env: NIGHTLY_STO_BLOB_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME) NIGHTLY_STO_ACC_SAS: $(NIGHTLY_STO_ACC_SAS) ACCOUNT_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net' VERBOSE_LOG: ${{ parameters.verbose_log }} continueOnError: false - script: cat $(BLOBFUSE2_CFG) displayName: "Print SAS config file" - template: 'azure-pipeline-templates/verify-auth.yml' parameters: working_dir: $(WORK_DIR) mount_dir: $(MOUNT_DIR) temp_dir: $(TEMP_DIR) idstring: 'Block SAS' distro_name: $(imageName) mountStep: script: | $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) --default-working-dir=$(WORK_DIR) displayName: 'AuthVerify-SAS: Mount' continueOnError: false # Block SPN and proxy test - script: | cd $(WORK_DIR) $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_spn_proxy.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG) displayName: "Create SPN Config File" env: NIGHTLY_SPN_ACC_NAME: $(AZTEST_BLOCK_ACC_NAME) NIGHTLY_SPN_CLIENT_ID: $(AZTEST_CLIENT) NIGHTLY_SPN_TENANT_ID: $(AZTEST_TENANT) NIGHTLY_SPN_CLIENT_SECRET: $(AZTEST_SECRET) ACCOUNT_ENDPOINT: 'https://$(AZTEST_BLOCK_ACC_NAME).blob.core.windows.net' VERBOSE_LOG: ${{ parameters.verbose_log }} continueOnError: false - script: cat $(BLOBFUSE2_CFG) displayName: "Print SPN Config File" - template: 'azure-pipeline-templates/verify-auth.yml' parameters: working_dir: $(WORK_DIR) mount_dir: $(MOUNT_DIR) temp_dir: $(TEMP_DIR) idstring: 'Block SPN' distro_name: $(imageName) mountStep: script: | $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) --default-working-dir=$(WORK_DIR) displayName: 'AuthVerify-SPN: Mount' continueOnError: false # ------------------------------------------------------------ - template: 'azure-pipeline-templates/cleanup.yml' parameters: working_dir: $(WORK_DIR) mount_dir: $(MOUNT_DIR) temp_dir: $(TEMP_DIR) - script: | kill -9 $(pgrep mitmdump) displayName: 'Kill Proxy' - ${{ if eq(parameters.exhaustive_test, true) }}: # RHEL Tests - job: Set_3 timeoutInMinutes: 60 strategy: matrix: RHEL-7.5: DistroVer: "RHEL-7.5" Description: "Red Hat Enterprise Linux 7.5" AgentName: "blobfuse-rhel7_5" ContainerName: "test-cnt-rhel-75" tags: 'fuse3' RHEL-8.1: DistroVer: "RHEL-8.1" Description: "Red Hat Enterprise Linux 8.1" AgentName: "blobfuse-rhel8_1" containerName: "test-cnt-rhel-81" tags: 'fuse3' RHEL-8.2: DistroVer: "RHEL-8.2" Description: "Red Hat Enterprise Linux 8.2" AgentName: "blobfuse-rhel8_2" containerName: "test-cnt-rhel-82" tags: 'fuse3' RHEL-8.6: DistroVer: "RHEL-8.6" Description: "Red Hat Enterprise Linux 8.6" AgentName: "blobfuse-rhel8_6" containerName: "test-cnt-rhel-86" tags: 'fuse3' # RHEL-9.0: # DistroVer: "RHEL-9.0" # Description: "Red Hat Enterprise Linux 9.0" # AgentName: "blobfuse-rhel9" # containerName: "test-cnt-rhel-9" # tags: 'fuse3' pool: name: "blobfuse-rhel-pool" demands: - ImageOverride -equals $(AgentName) variables: - group: NightlyBlobFuse - name: ROOT_DIR value: "/usr/pipeline/workv2" - name: WORK_DIR value: "/usr/pipeline/workv2/go/src/azure-storage-fuse" - name: skipComponentGovernanceDetection value: true - name: MOUNT_DIR value: "/usr/pipeline/workv2/blob_mnt" - name: TEMP_DIR value: "/usr/pipeline/workv2/temp" - name: BLOBFUSE2_CFG value: "/usr/pipeline/workv2/blobfuse2.yaml" - name: BLOBFUSE2_ADLS_CFG value: "/home/vsts/workv2/blobfuse2.adls.yaml" - name: GOPATH value: "/usr/pipeline/workv2/go" steps: # Go tool installer - task: GoTool@0 inputs: version: '1.16.2' displayName: "Install Go Version" - script: | sudo touch /etc/yum.repos.d/centos.repo sudo sh -c 'echo -e "[centos-extras]\nname=Centos extras - $basearch\nbaseurl=http://mirror.centos.org/centos/7/extras/x86_64\nenabled=1\ngpgcheck=1\ngpgkey=http://centos.org/keys/RPM-GPG-KEY-CentOS-7" > /etc/yum.repos.d/centos.repo' condition: eq(variables['AgentName'], 'blobfuse-rhel7_5') displayName: "Update OS mirrors" - template: 'azure-pipeline-templates/distro-tests.yml' parameters: working_dir: $(WORK_DIR) root_dir: $(ROOT_DIR) temp_dir: $(TEMP_DIR) mount_dir: $(MOUNT_DIR) config_path: $(BLOBFUSE2_CFG) container: $(ContainerName) blob_account_name: $(NIGHTLY_STO_BLOB_ACC_NAME) blob_account_key: $(NIGHTLY_STO_BLOB_ACC_KEY) adls_account_name: $(AZTEST_ADLS_ACC_NAME) adls_account_key: $(AZTEST_ADLS_KEY) distro_name: $(AgentName) gopath: $(GOPATH) tags: $(tags) installStep: script: | sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo sudo yum update -y sudo yum groupinstall "Development Tools" -y if [ $(AgentName) == "blobfuse-rhel7_5" ]; then sudo yum install git fuse fuse3-libs fuse3-devel fuse3 rh-python36 -y else sudo yum install git fuse fuse3-libs fuse3-devel fuse3 python36 -y --nobest --allowerasing fi displayName: 'Install fuse' verbose_log: ${{ parameters.verbose_log }} # Centos Tests - job: Set_4 timeoutInMinutes: 60 strategy: matrix: CentOS-7.9: DistroVer: "CentOS-7.9" Description: "CentOS 7.9" AgentName: "blobfuse-centos7" ContainerName: "test-cnt-cent-7" CentOS-8.5: DistroVer: "CentOS-8.5" Description: "CentOS 8.5" AgentName: "blobfuse-centos8" ContainerName: "test-cnt-cent-8" pool: name: "blobfuse-centos-pool" demands: - ImageOverride -equals $(AgentName) variables: - group: NightlyBlobFuse - name: ROOT_DIR value: "/usr/pipeline/workv2" - name: WORK_DIR value: "/usr/pipeline/workv2/go/src/azure-storage-fuse" - name: skipComponentGovernanceDetection value: true - name: MOUNT_DIR value: "/usr/pipeline/workv2/blob_mnt" - name: TEMP_DIR value: "/usr/pipeline/workv2/temp" - name: BLOBFUSE2_CFG value: "/usr/pipeline/workv2/blobfuse2.yaml" - name: BLOBFUSE2_ADLS_CFG value: "/home/vsts/workv2/blobfuse2.adls.yaml" - name: GOPATH value: "/usr/pipeline/workv2/go" steps: # Go tool installer - task: GoTool@0 inputs: version: '1.16.2' displayName: "Install Go Version" - script: | sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* sudo sed -i 's|baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* condition: eq(variables['AgentName'], 'blobfuse-centos8') displayName: "Update OS mirrors" - template: 'azure-pipeline-templates/distro-tests.yml' parameters: working_dir: $(WORK_DIR) root_dir: $(ROOT_DIR) temp_dir: $(TEMP_DIR) mount_dir: $(MOUNT_DIR) config_path: $(BLOBFUSE2_CFG) container: $(ContainerName) blob_account_name: $(NIGHTLY_STO_BLOB_ACC_NAME) blob_account_key: $(NIGHTLY_STO_BLOB_ACC_KEY) adls_account_name: $(AZTEST_ADLS_ACC_NAME) adls_account_key: $(AZTEST_ADLS_KEY) distro_name: $(AgentName) gopath: $(GOPATH) installStep: script: | sudo yum update -y --skip-broken if [ $(AgentName) == "blobfuse-centos8" ]; then sudo yum install gcc gcc-c++ make git fuse fuse3 fuse3-devel python36 -y --nobest --allowerasing else sudo yum install gcc gcc-c++ make git fuse3 fuse3-devel python36 -y fi displayName: 'Install fuse' verbose_log: ${{ parameters.verbose_log }} # Oracle Tests - job: Set_5 timeoutInMinutes: 60 strategy: matrix: Oracle-8.1: DistroVer: "Oracle-8.1" Description: "Oracle Linux 8.1" AgentName: "blobfuse-oracle81" ContainerName: "test-cnt-ora-81" pool: name: "blobfuse-oracle-pool" demands: - ImageOverride -equals $(AgentName) variables: - group: NightlyBlobFuse - name: ROOT_DIR value: "/usr/pipeline/workv2" - name: WORK_DIR value: "/usr/pipeline/workv2/go/src/azure-storage-fuse" - name: skipComponentGovernanceDetection value: true - name: MOUNT_DIR value: "/usr/pipeline/workv2/blob_mnt" - name: TEMP_DIR value: "/usr/pipeline/workv2/temp" - name: BLOBFUSE2_CFG value: "/usr/pipeline/workv2/blobfuse2.yaml" - name: BLOBFUSE2_ADLS_CFG value: "/home/vsts/workv2/blobfuse2.adls.yaml" - name: GOPATH value: "/usr/pipeline/workv2/go" steps: # Go tool installer - task: GoTool@0 inputs: version: '1.16.2' displayName: "Install Go Version" - template: 'azure-pipeline-templates/distro-tests.yml' parameters: working_dir: $(WORK_DIR) root_dir: $(ROOT_DIR) temp_dir: $(TEMP_DIR) mount_dir: $(MOUNT_DIR) config_path: $(BLOBFUSE2_CFG) container: $(ContainerName) blob_account_name: $(NIGHTLY_STO_BLOB_ACC_NAME) blob_account_key: $(NIGHTLY_STO_BLOB_ACC_KEY) adls_account_name: $(AZTEST_ADLS_ACC_NAME) adls_account_key: $(AZTEST_ADLS_KEY) distro_name: $(AgentName) gopath: $(GOPATH) installStep: script: | sudo yum update -y sudo yum install gcc gcc-c++ make git fuse fuse3 fuse3-devel python36 -y --nobest --allowerasing displayName: 'Install fuse' verbose_log: ${{ parameters.verbose_log }} - job: Set_6 timeoutInMinutes: 60 strategy: matrix: Debian-9.0: DistroVer: "Debian9.0" AgentName: "DEB 9.0" Description: "Debian Linux 9.0 Gen 1" containerName: 'test-cnt-deb-9' fuselib: 'libfuse-dev' tags: 'fuse2' pool: name: 'Blobfuse Pool' demands: - Agent.Name -equals $(AgentName) variables: - group: NightlyBlobFuse - name: ROOT_DIR value: "/home/vsts/workv2" - name: WORK_DIR value: "/home/vsts/workv2/go/src/azure-storage-fuse" - name: skipComponentGovernanceDetection value: true - name: MOUNT_DIR value: "/home/vsts/workv2/blob_mnt" - name: TEMP_DIR value: "/home/vsts/workv2/blobfuse2tmp" - name: BLOBFUSE2_CFG value: "/home/vsts/workv2/blobfuse2.yaml" - name: BLOBFUSE2_ADLS_CFG value: "/home/vsts/workv2/blobfuse2.adls.yaml" - name: GOPATH value: "/home/vsts/workv2/go" # Distro Tests steps: - template: 'azure-pipeline-templates/distro-tests.yml' parameters: working_dir: $(WORK_DIR) root_dir: $(ROOT_DIR) temp_dir: $(TEMP_DIR) mount_dir: $(MOUNT_DIR) gopath: $(GOPATH) config_path: $(BLOBFUSE2_CFG) container: $(containerName) blob_account_name: $(NIGHTLY_STO_BLOB_ACC_NAME) blob_account_key: $(NIGHTLY_STO_BLOB_ACC_KEY) adls_account_name: $(AZTEST_ADLS_ACC_NAME) adls_account_key: $(AZTEST_ADLS_KEY) distro_name: $(AgentName) tags: $(tags) fuselib: $(fuselib) installStep: script: | sudo apt-get update --fix-missing sudo apt-get install $(fuselib) -y displayName: 'Install libfuse' verbose_log: ${{ parameters.verbose_log }} # Debian Tests - job: Set_7 timeoutInMinutes: 60 strategy: matrix: # Debian-9.0: # DistroVer: "Debian9.0" # Description: "Debian 9" # AgentName: "blobfuse-debian9" # ContainerName: "test-cnt-deb-9" # fuselib: 'fuse libfuse-dev' # tags: 'fuse2' Debian-10.0: DistroVer: "Debian10.0" Description: "Debian 10" AgentName: "blobfuse-debian10" ContainerName: "test-cnt-deb-10" fuselib: 'fuse libfuse-dev' tags: 'fuse2' Debian-11.0: DistroVer: "Debian11.0" Description: "Debian 11" AgentName: "blobfuse-debian11" ContainerName: "test-cnt-deb-11" fuselib: 'fuse3 libfuse3-dev' tags: 'fuse3' pool: name: "blobfuse-debian-pool" demands: - ImageOverride -equals $(AgentName) variables: - group: NightlyBlobFuse - name: ROOT_DIR value: "/usr/pipeline/workv2" - name: WORK_DIR value: "/usr/pipeline/workv2/go/src/azure-storage-fuse" - name: skipComponentGovernanceDetection value: true - name: MOUNT_DIR value: "/usr/pipeline/workv2/blob_mnt" - name: TEMP_DIR value: "/usr/pipeline/workv2/temp" - name: BLOBFUSE2_CFG value: "/usr/pipeline/workv2/blobfuse2.yaml" - name: BLOBFUSE2_ADLS_CFG value: "/home/vsts/workv2/blobfuse2.adls.yaml" - name: GOPATH value: "/usr/pipeline/workv2/go" steps: # Go tool installer - task: GoTool@0 inputs: version: '1.16.2' displayName: "Install Go Version" - template: 'azure-pipeline-templates/distro-tests.yml' parameters: working_dir: $(WORK_DIR) root_dir: $(ROOT_DIR) temp_dir: $(TEMP_DIR) mount_dir: $(MOUNT_DIR) config_path: $(BLOBFUSE2_CFG) container: $(ContainerName) blob_account_name: $(NIGHTLY_STO_BLOB_ACC_NAME) blob_account_key: $(NIGHTLY_STO_BLOB_ACC_KEY) adls_account_name: $(AZTEST_ADLS_ACC_NAME) adls_account_key: $(AZTEST_ADLS_KEY) distro_name: $(AgentName) tags: $(tags) fuselib: $(fuselib) gopath: $(GOPATH) installStep: script: | sudo rm /etc/apt/sources.list.d/azure.list sudo apt-get update --fix-missing -y sudo apt-get install $(fuselib) -y sudo apt-get install build-essential git python3 -y displayName: 'Install fuse' verbose_log: ${{ parameters.verbose_log }} # SUSE Tests - job: Set_8 timeoutInMinutes: 60 strategy: matrix: SUSE-15: DistroVer: "SUSE-15" Description: "SUSE Enterprise Linux 15" AgentName: "blobfuse-suse15" ContainerName: "test-cnt-suse-15" pool: name: "blobfuse-suse-pool" demands: - ImageOverride -equals $(AgentName) variables: - group: NightlyBlobFuse - name: ROOT_DIR value: "/usr/pipeline/workv2" - name: WORK_DIR value: "/usr/pipeline/workv2/go/src/azure-storage-fuse" - name: skipComponentGovernanceDetection value: true - name: MOUNT_DIR value: "/usr/pipeline/workv2/blob_mnt" - name: TEMP_DIR value: "/usr/pipeline/workv2/temp" - name: BLOBFUSE2_CFG value: "/usr/pipeline/workv2/blobfuse2.yaml" - name: BLOBFUSE2_ADLS_CFG value: "/home/vsts/workv2/blobfuse2.adls.yaml" - name: GOPATH value: "/usr/pipeline/workv2/go" steps: # Go tool installer - task: GoTool@0 inputs: version: '1.16.2' displayName: "Install Go Version" - template: 'azure-pipeline-templates/distro-tests.yml' parameters: working_dir: $(WORK_DIR) root_dir: $(ROOT_DIR) temp_dir: $(TEMP_DIR) mount_dir: $(MOUNT_DIR) config_path: $(BLOBFUSE2_CFG) container: $(ContainerName) blob_account_name: $(NIGHTLY_STO_BLOB_ACC_NAME) blob_account_key: $(NIGHTLY_STO_BLOB_ACC_KEY) adls_account_name: $(AZTEST_ADLS_ACC_NAME) adls_account_key: $(AZTEST_ADLS_KEY) distro_name: $(AgentName) gopath: $(GOPATH) installStep: script: | sudo zypper -n install git golang make cmake gcc gcc-c++ glibc-devel fuse fuse3 wget https://rpmfind.net/linux/opensuse/distribution/leap/15.2/repo/oss/x86_64/fuse3-devel-3.6.1-lp152.1.19.x86_64.rpm sudo zypper -n --no-gpg-checks install fuse3-devel-3.6.1-lp152.1.19.x86_64.rpm displayName: 'Install fuse' verbose_log: ${{ parameters.verbose_log }} # Mariner Tests - job: Set_9 timeoutInMinutes: 60 strategy: matrix: Mariner: DistroVer: "Mariner" Description: "CBL-Mariner Linux" AgentName: "blobfuse-mariner" ContainerName: "test-cnt-mari-1" fuselib: 'libfuse-dev' tags: 'fuse2' pool: name: "blobfuse-mariner-pool" demands: - ImageOverride -equals $(AgentName) variables: - group: NightlyBlobFuse - name: ROOT_DIR value: "/usr/pipeline/workv2" - name: WORK_DIR value: "/usr/pipeline/workv2/go/src/azure-storage-fuse" - name: skipComponentGovernanceDetection value: true - name: MOUNT_DIR value: "/usr/pipeline/workv2/blob_mnt" - name: TEMP_DIR value: "/usr/pipeline/workv2/temp" - name: BLOBFUSE2_CFG value: "/usr/pipeline/workv2/blobfuse2.yaml" - name: BLOBFUSE2_ADLS_CFG value: "/home/vsts/workv2/blobfuse2.adls.yaml" - name: GOPATH value: "/usr/pipeline/workv2/go" steps: # Go tool installer - task: GoTool@0 inputs: version: '1.16.2' displayName: "Install Go Version" - template: 'azure-pipeline-templates/distro-tests.yml' parameters: working_dir: $(WORK_DIR) root_dir: $(ROOT_DIR) temp_dir: $(TEMP_DIR) mount_dir: $(MOUNT_DIR) config_path: $(BLOBFUSE2_CFG) container: $(ContainerName) blob_account_name: $(NIGHTLY_STO_BLOB_ACC_NAME) blob_account_key: $(NIGHTLY_STO_BLOB_ACC_KEY) adls_account_name: $(AZTEST_ADLS_ACC_NAME) adls_account_key: $(AZTEST_ADLS_KEY) distro_name: $(AgentName) tags: $(tags) fuselib: $(fuselib) gopath: $(GOPATH) installStep: script: | sudo tdnf install build-essential git fuse fuse-devel python36 -y displayName: 'Install fuse' verbose_log: ${{ parameters.verbose_log }} - ${{ if eq(parameters.msi_test, true) }}: # ----------------------------------------------------------- # Ubuntu-20.04 MSI tests - job: Set_10 timeoutInMinutes: 60 strategy: matrix: Ubuntu-20-MSI: DistroVer: "Ubn20_MSI" AgentName: "blobfuse-ubuntu20" Description: "Ubuntu 20 MSI Test" pool: name: "blobfuse-ubuntu-pool" demands: - ImageOverride -equals $(AgentName) variables: - group: NightlyBlobFuse - name: ROOT_DIR value: "/home/vsts/workv2/" - name: WORK_DIR value: "/home/vsts/workv2/go/src/azure-storage-fuse" - name: skipComponentGovernanceDetection value: true - name: MOUNT_DIR value: "/home/vsts/workv2/blob_mnt" - name: TEMP_DIR value: "/home/vsts/workv2/blobfuse2tmp" - name: BLOBFUSE2_CFG value: "/home/vsts/workv2//myblobv2.msi.yaml" - name: BLOBFUSE2_CFG_ADLS value: "/home/vsts/workv2/myblobv2.msi.adls.yaml" - name: GOPATH value: "/home/vsts/workv2/go" - name: containerName value: "test-cnt-ubn-20-msi" steps: - checkout: none # Print the agent info - script: | echo $(Description) hostnamectl displayName: 'Print Agent Info' # Install libfuse - script: | sudo apt-get install make cmake gcc g++ libfuse3-dev fuse3 -y -o Dpkg::Options::="--force-confnew" sudo apt-get update --fix-missing -o Dpkg::Options::="--force-confnew" displayName: 'Install Fuse' # Prestart cleanup - script: | sudo fusermount3 -u $(MOUNT_DIR) sudo kill -9 `pidof blobfuse2` sudo rm -rf $(ROOT_DIR) displayName: 'PreBuild Cleanup' # Create directory structure - script: | sudo rm -rf $(ROOT_DIR) sudo mkdir -p $(ROOT_DIR) sudo chown -R `whoami` $(ROOT_DIR) chmod 777 $(ROOT_DIR) mkdir -p $(ROOT_DIR)/go/src displayName: 'Create Directory Structure' # Checkout the code - script: | git clone https://github.com/Azure/azure-storage-fuse displayName: 'Checkout Code' workingDirectory: $(ROOT_DIR)/go/src # Pull the branch - script: | git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` displayName: 'Checkout Branch' workingDirectory: $(WORK_DIR) # Build the code - template: 'azure-pipeline-templates/build.yml' parameters: working_directory: $(WORK_DIR) root_dir: $(ROOT_DIR) mount_dir: $(MOUNT_DIR) temp_dir: $(TEMP_DIR) gopath: $(GOPATH) container: $(containerName) skip_msi: "false" # BlockBlob MSI Test - script: | cd $(WORK_DIR) $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_msi.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG) displayName: "Create MSI Config File" env: NIGHTLY_STO_BLOB_ACC_NAME: $(AZTEST_BLOCK_ACC_NAME) NIGHTLY_MSI_APP_ID: $(AZTEST_APP_ID) ACCOUNT_TYPE: 'block' ACCOUNT_ENDPOINT: 'https://$(AZTEST_BLOCK_ACC_NAME).blob.core.windows.net' VERBOSE_LOG: ${{ parameters.verbose_log }} continueOnError: false - script: cat $(BLOBFUSE2_CFG) displayName: "Print config file" # Mount the cntainer and run basic tests - template: 'azure-pipeline-templates/verify-auth.yml' parameters: working_dir: $(WORK_DIR) mount_dir: $(MOUNT_DIR) temp_dir: $(TEMP_DIR) idstring: 'BlockBlob MSI' distro_name: $(AgentName) mountStep: script: | $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) displayName: 'AuthVerify MSI: Mount Container' continueOnError: false # ADLS MSI Test - script: | cd $(WORK_DIR) $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_msi.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG) displayName: "Create MSI Config File" env: NIGHTLY_STO_BLOB_ACC_NAME: $(AZTEST_ADLS_ACC_NAME) NIGHTLY_MSI_APP_ID: $(AZTEST_APP_ID) ACCOUNT_TYPE: 'adls' ACCOUNT_ENDPOINT: 'https://$(AZTEST_ADLS_ACC_NAME).dfs.core.windows.net' VERBOSE_LOG: ${{ parameters.verbose_log }} continueOnError: false - script: cat $(BLOBFUSE2_CFG) displayName: "Print config file" # Mount the cntainer and run basic tests - template: 'azure-pipeline-templates/verify-auth.yml' parameters: working_dir: $(WORK_DIR) mount_dir: $(MOUNT_DIR) temp_dir: $(TEMP_DIR) idstring: 'ADLS MSI' distro_name: $(AgentName) mountStep: script: | $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) displayName: 'AuthVerify MSI: Mount Container' continueOnError: false # Cleanup - template: 'azure-pipeline-templates/cleanup.yml' parameters: working_dir: $(WORK_DIR) mount_dir: $(MOUNT_DIR) temp_dir: $(TEMP_DIR) - ${{ if eq(parameters.data_validation, true) }}: - stage: DataValidation jobs: # Ubuntu Tests - job: Set_1 timeoutInMinutes: 300 strategy: matrix: Ubuntu-20: imageName: 'ubuntu-20.04' containerName: 'test-cnt-ubn-20' adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_20) fuselib: 'libfuse-dev' tags: 'fuse2' Ubuntu-22: imageName: 'ubuntu-22.04' containerName: 'test-cnt-ubn-22' adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22) fuselib: 'libfuse3-dev' tags: 'fuse3' pool: vmImage: $(imageName) variables: - group: NightlyBlobFuse - name: ROOT_DIR value: "/usr/pipeline/workv2" - name: WORK_DIR value: "/usr/pipeline/workv2/go/src/azure-storage-fuse" - name: skipComponentGovernanceDetection value: true - name: MOUNT_DIR value: "/usr/pipeline/workv2/blob_mnt" - name: TEMP_DIR value: "/usr/pipeline/workv2/temp" - name: BLOBFUSE2_CFG value: "/usr/pipeline/workv2/blobfuse2.yaml" - name: BLOBFUSE2_STREAM_CFG value: "/usr/pipeline/workv2/blobfuse2_stream.yaml" - name: BLOBFUSE2_ADLS_CFG value: "/home/vsts/workv2/blobfuse2.adls.yaml" - name: GOPATH value: "/usr/pipeline/workv2/go" steps: - checkout: none - template: 'azure-pipeline-templates/distro-tests.yml' parameters: working_dir: $(WORK_DIR) root_dir: $(ROOT_DIR) temp_dir: $(TEMP_DIR) mount_dir: $(MOUNT_DIR) config_path: $(BLOBFUSE2_CFG) stream_config: $(BLOBFUSE2_STREAM_CFG) container: $(ContainerName) blob_account_name: $(NIGHTLY_STO_BLOB_ACC_NAME) blob_account_key: $(NIGHTLY_STO_BLOB_ACC_KEY) adls_account_name: $(AZTEST_ADLS_ACC_NAME) adls_account_key: $(AZTEST_ADLS_KEY) distro_name: $(AgentName) test_stream: true quick_test: false clone: true gopath: $(GOPATH) tags: $(tags) installStep: script: | sudo apt-get update --fix-missing sudo apt-get install $(fuselib) -y displayName: 'Install fuse' verbose_log: ${{ parameters.verbose_log }} - ${{ if eq(parameters.healthmon_test, true) }}: - stage: Healthmon jobs: - job: Set_1 timeoutInMinutes: 300 strategy: matrix: Ubuntu-22: imageName: 'ubuntu-22.04' containerName: 'test-cnt-ubn-22' adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22) fuselib: 'libfuse3-dev' tags: 'fuse3' pool: vmImage: $(imageName) variables: - group: NightlyBlobFuse - name: MOUNT_DIR value: '$(Pipeline.Workspace)/blob_mnt' - name: TEMP_DIR value: '$(Pipeline.Workspace)/blobfuse2_tmp' - name: BLOBFUSE2_CFG value: '$(Pipeline.Workspace)/blobfuse2.yaml' - name: GOPATH value: '$(Pipeline.Workspace)/go' - name: ROOT_DIR value: '$(System.DefaultWorkingDirectory)' - name: WORK_DIR value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' steps: - checkout: none - script: | sudo apt-get update --fix-missing -o Dpkg::Options::="--force-confnew" sudo apt-get install make cmake gcc g++ parallel $(fuselib) -y -o Dpkg::Options::="--force-confnew" displayName: 'Install libfuse' # Clone the repo - script: | git clone https://github.com/Azure/azure-storage-fuse displayName: 'Checkout Code' workingDirectory: $(ROOT_DIR) # Checkout the branch - script: | git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` displayName: 'Checkout Branch' workingDirectory: $(WORK_DIR) # Create directory structure - script: | sudo mkdir -p $(ROOT_DIR) sudo chown -R `whoami` $(ROOT_DIR) chmod 777 $(ROOT_DIR) displayName: 'Create Directory Structure' # ------------------------------------------------------- # Alternative custom script for Self-Hosted agents to install Go-lang - task: ShellScript@2 inputs: scriptPath: "$(WORK_DIR)/go_installer.sh" args: "$(ROOT_DIR)/ 1.16.2" displayName: "GoTool Custom Setup" # Downloading Go dependency packages - task: Go@0 inputs: command: 'get' arguments: '-d' workingDirectory: $(WORK_DIR) displayName: "Go Get" # Building our package - task: Go@0 inputs: command: 'build' arguments: "-o blobfuse2" workingDirectory: $(WORK_DIR) displayName: "Go Build Blobfuse2" # Building our package - task: Go@0 inputs: command: 'build' arguments: "-o bfusemon ./tools/health-monitor/" workingDirectory: $(WORK_DIR) displayName: "Go Build bfusemon" - script: | cd $(WORK_DIR) $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key_hmon.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG) displayName: Create Key Config File env: ACCOUNT_TYPE: 'block' NIGHTLY_STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME) NIGHTLY_STO_ACC_KEY: $(NIGHTLY_STO_BLOB_ACC_KEY) ACCOUNT_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net' VERBOSE_LOG: false USE_HTTP: false HMON_OUTPUT: $(WORK_DIR) continueOnError: false - script: cat $(BLOBFUSE2_CFG) displayName: Print config file - script: mkdir -p $(MOUNT_DIR) mkdir -p $(TEMP_DIR) displayName: "Create directory structure" - template: 'azure-pipeline-templates/mount.yml' parameters: working_dir: $(WORK_DIR) mount_dir: $(MOUNT_DIR) temp_dir: $(TEMP_DIR) prefix: "Mount" mountStep: script: | sudo ln -s `pwd`/bfusemon /usr/local/bin/bfusemon ls -l /usr/local/bin/bfusemon* $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) displayName: 'Mount Blobfuse2' workingDirectory: $(WORK_DIR) timeoutInMinutes: 3 continueOnError: false - script: | touch $(MOUNT_DIR)/h1.txt echo "123456" > $(MOUNT_DIR)/h2.txt cp *.md $(MOUNT_DIR)/ mkdir $(MOUNT_DIR)/A mv $(MOUNT_DIR)/h1.txt $(MOUNT_DIR)/h5.txt mv $(MOUNT_DIR)/h*.* $(MOUNT_DIR)/A rm -rf $(MOUNT_DIR)/*.md rm -rf $(MOUNT_DIR)/A displayName: "Test for health monitor" workingDirectory: $(WORK_DIR) - script: | sleep 20s cat $(WORK_DIR)/monitor_*.json displayName: "Print bfusemon output" workingDirectory: $(WORK_DIR) - script: | sudo ps -aux | grep bfusemon displayName: 'Print bfusemon Process info' env: mount_dir: $(MOUNT_DIR) - template: 'azure-pipeline-templates/cleanup.yml' parameters: working_dir: $(WORK_DIR) mount_dir: $(MOUNT_DIR) temp_dir: $(TEMP_DIR)