Azure
/
azure-storage-fuse
зеркало из https://github.com/Azure/azure-storage-fuse.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность
azure-storage-fuse/blobfuse2-code-coverage.yaml

369 строки
14 KiB
YAML
Исходник Ответственный История

# Blobfuse2 Nightly Build-Sanity Pipeline
# In case of failure on a Self-Hosted Agent perform the following steps to get the vm back online:
# 1. Check which vm is offline by going to agent-pools in Azure pipelines portal
# 2. Log into the VM that is offline
# 3. Clear the _work or work directory which must be in myagent or $(HOME) directory
# 4. Verify whether system is online from the Azure pipelines portal
jobs:
- job: Coverage
timeoutInMinutes: 300
strategy:
matrix:
Ubuntu-18:
imageName: 'ubuntu-18.04'
containerName: 'test-cnt-ubn-18'
fuselib: 'libfuse-dev'
tags: 'fuse2'
hostedAgent: true
stressParallel: 3
Ubuntu-20:
imageName: 'ubuntu-20.04'
containerName: 'test-cnt-coverage'
fuselib: 'libfuse3-dev'
tags: 'fuse3'
hostedAgent: true
stressParallel: 1
pool:
vmImage: $(imageName)
variables:
- group: NightlyBlobFuse
- name: MOUNT_DIR
value: '$(Pipeline.Workspace)/blob_mnt'
- name: TEMP_DIR
value: '$(Pipeline.Workspace)/blobfuse2_tmp'
- name: BLOBFUSE2_CFG
value: '$(Pipeline.Workspace)/blobfuse2.yaml'
- name: BLOBFUSE2_STREAM_CFG
value: '$(Pipeline.Workspace)/blobfuse2_stream.yaml'
- name: BLOBFUSE2_ADLS_CFG
value: '$(Pipeline.Workspace)/blobfuse2.adls.yaml'
- name: GOPATH
value: '$(Pipeline.Workspace)/go'
- name: ROOT_DIR
value: '$(System.DefaultWorkingDirectory)'
- name: WORK_DIR
value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse'
steps:
- checkout: none
# Clone the repo
- script: |
git clone https://github.com/Azure/azure-storage-fuse
displayName: 'Checkout Code'
workingDirectory: $(ROOT_DIR)
# Checkout the branch
- script: |
git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`
displayName: 'Checkout Branch'
workingDirectory: $(WORK_DIR)
- script: |
sudo apt-get update --fix-missing
sudo apt-get install $(fuselib) -y
displayName: 'Install libfuse'
# -------------------------------------------------------
# Pull and build the code
- template: 'azure-pipeline-templates/build.yml'
parameters:
working_directory: $(WORK_DIR)
root_dir: $(Pipeline.Workspace)
mount_dir: $(MOUNT_DIR)
temp_dir: $(TEMP_DIR)
gopath: $(GOPATH)
container: $(containerName)
tags: $(tags)
fuselib: $(fuselib)
# -------------------------------------------------------
# UT based code coverage test
- script: |
go test -v -timeout=1h ./... --tags=unittest,$(tags) -covermode count -coverprofile ./blobfuse2_ut.cov
workingDirectory: $(WORK_DIR)
displayName: "UT Code Coverage"
# -------------------------------------------------------
# Build code coverage binary
- script: |
go test -coverpkg="./..." -covermode=count -c --tags=$(tags) -o blobfuse2.test
workingDirectory: $(WORK_DIR)
displayName: "Build coverage binary"
# -------------------------------------------------------
# Config Generation (Block Blob)
- script: |
cd $(WORK_DIR)
$(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG)
env:
NIGHTLY_STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME)
NIGHTLY_STO_ACC_KEY: $(NIGHTLY_STO_BLOB_ACC_KEY)
ACCOUNT_TYPE: 'block'
ACCOUNT_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net'
displayName: 'Create Config File - Block Blob'
workingDirectory: $(WORK_DIR)
continueOnError: false
# Code Coverage with e2e-tests for block blob
- script: |
rm -rf $(MOUNT_DIR)/*
rm -rf $(TEMP_DIR)/*
./blobfuse2.test -test.v -test.coverprofile=blobfuse2_block.cov mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) --foreground=true &
sleep 10
ps -aux | grep blobfuse2
rm -rf $(MOUNT_DIR)/*
go test -v -timeout=7200s test/e2e_tests -args -mnt-path=$(MOUNT_DIR)
sudo fusermount -u $(MOUNT_DIR)
sleep 5
workingDirectory: $(WORK_DIR)
displayName: "Block Blob Coverage"
- script: |
cd $(WORK_DIR)
$(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_ADLS_CFG)
env:
NIGHTLY_STO_ACC_NAME: $(AZTEST_ADLS_ACC_NAME)
NIGHTLY_STO_ACC_KEY: $(AZTEST_ADLS_KEY)
ACCOUNT_TYPE: 'adls'
ACCOUNT_ENDPOINT: 'https://$(AZTEST_ADLS_ACC_NAME).dfs.core.windows.net'
displayName: 'Create Config File - ADLS'
continueOnError: false
# Code Coverage with e2e-tests for adls blob
- script: |
rm -rf $(MOUNT_DIR)/*
rm -rf $(TEMP_DIR)/*
./blobfuse2.test -test.v -test.coverprofile=blobfuse2_adls.cov mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_ADLS_CFG) --foreground=true &
sleep 10
ps -aux | grep blobfuse2
rm -rf $(MOUNT_DIR)/*
go test -v -timeout=7200s test/e2e_tests -args -mnt-path=$(MOUNT_DIR) -adls=true
./blobfuse2 unmount all
sleep 5
workingDirectory: $(WORK_DIR)
displayName: "ADLS Coverage"
# -------------------------------------------------------
# Config Generation (Block Blob - LFU policy)
- script: |
cd $(WORK_DIR)
$(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key_lfu.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG)
env:
NIGHTLY_STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME)
NIGHTLY_STO_ACC_KEY: $(NIGHTLY_STO_BLOB_ACC_KEY)
ACCOUNT_TYPE: 'block'
ACCOUNT_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net'
displayName: 'Create Config File - LFU'
continueOnError: false
# Code Coverage with e2e-tests for block blob with lfu policy
- script: |
rm -rf $(MOUNT_DIR)/*
rm -rf $(TEMP_DIR)/*
./blobfuse2.test -test.v -test.coverprofile=blobfuse2_block_lfu.cov mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) --foreground=true &
sleep 10
ps -aux | grep blobfuse2
rm -rf $(MOUNT_DIR)/*
go test -v -timeout=7200s test/e2e_tests -args -mnt-path=$(MOUNT_DIR)
./blobfuse2 unmount $(MOUNT_DIR)
sleep 5
workingDirectory: $(WORK_DIR)
displayName: "Block Blob LFU Coverage"
# -------------------------------------------------------
# Config Generation (Block Blob - Stream)
- script: |
cd $(WORK_DIR)
$(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_stream.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_STREAM_CFG)
displayName: 'Create Config File - Stream'
env:
NIGHTLY_STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME)
NIGHTLY_STO_ACC_KEY: $(NIGHTLY_STO_BLOB_ACC_KEY)
ACCOUNT_TYPE: 'block'
ACCOUNT_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net'
continueOnError: false
# Streaming test preparation
- script: |
rm -rf $(MOUNT_DIR)/*
rm -rf $(TEMP_DIR)/*
./blobfuse2.test -test.v -test.coverprofile=blobfuse2_stream_prep.cov mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) --foreground=true &
sleep 10
ps -aux | grep blobfuse2
for i in {10,50,100,200,500,1024}; do echo $i; done | parallel --will-cite -j 5 'head -c {}M < /dev/urandom > $(WORK_DIR)/myfile_{}'
for i in {10,50,100,200,500,1024}; do echo $i; done | parallel --will-cite -j 5 'cp $(WORK_DIR)/myfile_{} $(MOUNT_DIR)/'
./blobfuse2 unmount $(MOUNT_DIR)*
sudo fusermount -u $(MOUNT_DIR)
sleep 5
workingDirectory: $(WORK_DIR)
displayName: "Block Blob Stream Preparation"
# Code Coverage with e2e-tests for block blob with streaming on
- script: |
rm -rf $(MOUNT_DIR)/*
rm -rf $(TEMP_DIR)/*
./blobfuse2.test -test.v -test.coverprofile=blobfuse2_stream.cov mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_STREAM_CFG) --foreground=true &
sleep 10
ps -aux | grep blobfuse2
./blobfuse2 mount list
for i in {10,50,100,200,500,1024}; do echo $i; done | parallel --will-cite -j 5 'diff $(WORK_DIR)/myfile_{} $(MOUNT_DIR)/myfile_{}'
sudo fusermount -u $(MOUNT_DIR)
sleep 5
workingDirectory: $(WORK_DIR)
displayName: "Block Blob Stream Coverage"
# -------------------------------------------------------
# Config Generation (Block Blob) for cli options
- script: |
cd $(WORK_DIR)
$(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG)
displayName: 'Create Config File - Block Blob'
env:
NIGHTLY_STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME)
NIGHTLY_STO_ACC_KEY: $(NIGHTLY_STO_BLOB_ACC_KEY)
ACCOUNT_TYPE: 'block'
ACCOUNT_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net'
# Component generation code coverage
- script: |
./blobfuse2.test -test.v -test.coverprofile=generate_cmd.cov generate test_component
if [ $? -ne 0 ]; then
exit 1
fi
workingDirectory: $(WORK_DIR)
displayName: "CLI : Component Generation"
# Mount / Unmount / List code coverage
- script: |
rm -rf $(MOUNT_DIR)/*
rm -rf $(TEMP_DIR)/*
./blobfuse2.test -test.v -test.coverprofile=list_empty_cmd.cov mount list
if [ $? -ne 0 ]; then
exit 1
fi
workingDirectory: $(WORK_DIR)
displayName: "CLI : Mount List"
- script: |
./blobfuse2.test -test.v -test.coverprofile=mount_cmd.cov mount all $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) --log-level=log_debug
if [ $? -ne 0 ]; then
exit 1
fi
sleep 20
./blobfuse2.test -test.v -test.coverprofile=list_cmd_all.cov mount list
if [ $? -ne 0 ]; then
exit 1
fi
./blobfuse2 unmount all
workingDirectory: $(WORK_DIR)
displayName: "CLI : Mount all and List"
- script: |
./blobfuse2.test -test.v -test.coverprofile=mount_cmd_all.cov mount all $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) --log-level=log_debug
if [ $? -ne 0 ]; then
exit 1
fi
sleep 20
./blobfuse2.test -test.v -test.coverprofile=umnt_cmd_cont.cov unmount $(MOUNT_DIR)/$(containerName)
if [ $? -ne 0 ]; then
exit 1
fi
./blobfuse2.test -test.v -test.coverprofile=umnt_wild_cmd.cov unmount testmut*
if [ $? -ne 0 ]; then
exit 1
fi
./blobfuse2.test -test.v -test.coverprofile=umnt_negative_cmd.cov unmount abcdef
if [ $? -ne 0 ]; then
exit 1
fi
for i in {1..5}; do ./blobfuse2.test -test.v -test.coverprofile=umnt_all_cmd.cov unmount all; done
workingDirectory: $(WORK_DIR)
displayName: "CLI : Unmount options"
# Mount / Unmount Negative tests
- script: |
./blobfuse2.test -test.v -test.coverprofile=mount_neg.cov mount all /abc --config-file=$(BLOBFUSE2_CFG) --log-level=log_debug
if [ $? -eq 0 ]; then
exit 1
fi
workingDirectory: $(WORK_DIR)
displayName: "CLI : Invalid mount path"
timeoutInMinutes: 2
- script: |
./blobfuse2 unmount all
./blobfuse2.test -test.v -test.coverprofile=mount_foreg.cov mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) --log-level=log_debug --foreground=true &
if [ $? -ne 0 ]; then
exit 1
fi
sleep 5
./blobfuse2.test -test.v -test.coverprofile=mount_remount.cov mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) --log-level=log_debug
if [ $? -eq 0 ]; then
exit 1
fi
sleep 5
./blobfuse2 unmount all
workingDirectory: $(WORK_DIR)
displayName: "CLI : Remount test"
timeoutInMinutes: 2
# Simulate config change
- script: |
rm -rf $(MOUNT_DIR)/*
rm -rf $(TEMP_DIR)/*
./blobfuse2 unmount all
./blobfuse2.test -test.v -test.coverprofile=mount_foreg_2.cov mount all $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) --log-level=log_debug --foreground=true &
if [ $? -ne 0 ]; then
exit 1
fi
sleep 5
echo -e "\n\nlogging:\n level:log_debug\n type:base" >> $(BLOBFUSE2_CFG)
sed -i 's/timeout:.*/timeout: 300/' $(BLOBFUSE2_CFG)
sleep 5
./blobfuse2 unmount all
workingDirectory: $(WORK_DIR)
displayName: "CLI : Config change simulator"
# -------------------------------------------------------
# Coverage report consolidation
- script: |
echo 'mode: count' > ./blobfuse2_coverage_raw.rpt
tail -q -n +2 ./*.cov >> ./blobfuse2_coverage_raw.rpt
cat ./blobfuse2_coverage_raw.rpt | grep -v mock_component | grep -v base_component | grep -v loopback | grep -v "common/log" > ./blobfuse2_coverage.rpt
go tool cover -func blobfuse2_coverage.rpt > ./blobfuse2_func_cover.rpt
go tool cover -html=./blobfuse2_coverage.rpt -o ./blobfuse2_coverage.html
go tool cover -html=./blobfuse2_ut.cov -o ./blobfuse2_ut.html
cp ./*.rpt $(Build.ArtifactStagingDirectory)/
cp ./*.cov $(Build.ArtifactStagingDirectory)/
cp ./*.html $(Build.ArtifactStagingDirectory)/
workingDirectory: $(WORK_DIR)
displayName: "Save coverage report"
# Publish the Artifacts
- task: PublishBuildArtifacts@1
inputs:
artifactName: 'Blobfuse2 Coverage $(tags)'
displayName: 'Publish Artifacts for blobfuse2 code coverage'
condition: succeeded()
Ссылка в новой задаче Показать git blame Копировать постоянную ссылку