Data validation tests for multiple files in parallel (#772)
Adding test cases for data validation on multiple files in parallel
This commit is contained in:
Sourav Gupta
GitHub
Родитель
3856dcb1f0
Коммит
a3326af3dc
|
|
@ -31,6 +31,9 @@ parameters:
|
|||
- name: fuselib
|
||||
type: string
|
||||
default: "libfuse3-dev"
|
||||
- name: quick_test
|
||||
type: boolean
|
||||
default: "true"
|
||||
|
||||
steps:
|
||||
- checkout: none
|
||||
|
|
@ -108,6 +111,7 @@ steps:
|
|||
adls: false
|
||||
idstring: 'Distro BlockBlob with Key Credentials'
|
||||
distro_name: ${{ parameters.distro_name }}
|
||||
quick_test: ${{ parameters.quick_test }}
|
||||
mountStep:
|
||||
script: |
|
||||
${{ parameters.working_dir }}/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.config_path }}
|
||||
|
|
@ -152,6 +156,7 @@ steps:
|
|||
adls: true
|
||||
idstring: 'Distro ADLS with Key Credentials'
|
||||
distro_name: ${{ parameters.distro_name }}
|
||||
quick_test: ${{ parameters.quick_test }}
|
||||
mountStep:
|
||||
script: |
|
||||
${{ parameters.working_dir }}/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.config_path }}
|
||||
|
|
|
|||
|
|
@ -22,6 +22,9 @@ parameters:
|
|||
- name: account_endpoint
|
||||
- name: distro_name
|
||||
type: string
|
||||
- name: quick_test
|
||||
type: boolean
|
||||
default: "true"
|
||||
|
||||
steps:
|
||||
- script: |
|
||||
|
|
@ -47,6 +50,7 @@ steps:
|
|||
adls: ${{ parameters.adls }}
|
||||
idstring: '${{ parameters.idstring }}'
|
||||
distro_name: ${{ parameters.distro_name }}
|
||||
quick_test: ${{ parameters.quick_test }}
|
||||
mountStep:
|
||||
script: |
|
||||
$(WORK_DIR)/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.config_file }} --default-working-dir=$(System.DefaultWorkingDirectory)
|
||||
|
|
|
|||
|
|
@ -16,6 +16,9 @@ parameters:
|
|||
- name: clone
|
||||
type: boolean
|
||||
default: "false"
|
||||
- name: quick_test
|
||||
type: boolean
|
||||
default: "true"
|
||||
|
||||
steps:
|
||||
|
||||
|
|
@ -40,7 +43,7 @@ steps:
|
|||
- task: Go@0
|
||||
inputs:
|
||||
command: 'test'
|
||||
arguments: '-v -timeout=2h ./... -args -mnt-path=${{ parameters.mount_dir }} -adls=${{parameters.adls}} -clone=${{parameters.clone}} -tmp-path=${{parameters.temp_dir}}'
|
||||
arguments: '-v -timeout=2h ./... -args -mnt-path=${{ parameters.mount_dir }} -adls=${{parameters.adls}} -clone=${{parameters.clone}} -tmp-path=${{parameters.temp_dir}} -quick-test=${{parameters.quick_test}} -distro-name="${{parameters.distro_name}}"'
|
||||
workingDirectory: ${{ parameters.working_dir }}/test/e2e_tests
|
||||
displayName: 'E2E Test: ${{ parameters.idstring }}'
|
||||
timeoutInMinutes: 120
|
||||
|
|
@ -49,7 +52,7 @@ steps:
|
|||
mount_dir: ${{ parameters.mount_dir }}
|
||||
|
||||
- script: |
|
||||
cat blobfuse2-logs.txt
|
||||
tail -n 200 blobfuse2-logs.txt
|
||||
displayName: 'View Logs'
|
||||
condition: always()
|
||||
|
||||
|
|
|
|||
|
|
@ -51,6 +51,9 @@ parameters:
|
|||
type: string
|
||||
- name: distro_name
|
||||
type: string
|
||||
- name: quick_test
|
||||
type: boolean
|
||||
default: "true"
|
||||
|
||||
#--------------------------------------- Setup: End to end tests with different Storage configurations ------------------------------------------
|
||||
# Create key credential config file if we need to test it
|
||||
|
|
@ -116,6 +119,7 @@ steps:
|
|||
idstring: ${{ parameters.service }} with Key Credentials
|
||||
distro_name: ${{ parameters.distro_name }}
|
||||
clone: "true"
|
||||
quick_test: ${{ parameters.quick_test }}
|
||||
mountStep:
|
||||
script: >
|
||||
${{ parameters.working_dir }}/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.config }}
|
||||
|
|
@ -133,6 +137,7 @@ steps:
|
|||
adls: ${{ parameters.adls }}
|
||||
idstring: ${{ parameters.service }} with SAS Credentials
|
||||
distro_name: ${{ parameters.distro_name }}
|
||||
quick_test: ${{ parameters.quick_test }}
|
||||
mountStep:
|
||||
script: >
|
||||
${{ parameters.working_dir }}/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.sas_credential_config }}
|
||||
|
|
@ -150,6 +155,7 @@ steps:
|
|||
adls: ${{ parameters.adls }}
|
||||
idstring: ${{ parameters.service }} with SPN Credentials
|
||||
distro_name: ${{ parameters.distro_name }}
|
||||
quick_test: ${{ parameters.quick_test }}
|
||||
mountStep:
|
||||
script: >
|
||||
${{ parameters.working_dir }}/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.spn_credential_config }}
|
||||
|
|
@ -174,6 +180,7 @@ steps:
|
|||
account_endpoint: ${{ parameters.account_endpoint }}
|
||||
idstring: "${{ parameters.service }} LFU policy"
|
||||
distro_name: ${{ parameters.distro_name }}
|
||||
quick_test: ${{ parameters.quick_test }}
|
||||
|
||||
- template: e2e-tests-spcl.yml
|
||||
parameters:
|
||||
|
|
@ -189,6 +196,7 @@ steps:
|
|||
account_endpoint: ${{ parameters.account_endpoint }}
|
||||
idstring: "${{ parameters.service }} LRU policy no timeout"
|
||||
distro_name: ${{ parameters.distro_name }}
|
||||
quick_test: ${{ parameters.quick_test }}
|
||||
|
||||
- template: e2e-tests-spcl.yml
|
||||
parameters:
|
||||
|
|
@ -204,6 +212,7 @@ steps:
|
|||
account_endpoint: ${{ parameters.account_endpoint }}
|
||||
idstring: "${{ parameters.service }} LRU policy create empty"
|
||||
distro_name: ${{ parameters.distro_name }}
|
||||
quick_test: ${{ parameters.quick_test }}
|
||||
|
||||
|
||||
#--------------------------------------- Setup: End to end tests with different File Cache configurations ------------------------------------------
|
||||
|
|
|
|||
|
|
@ -173,6 +173,7 @@ jobs:
|
|||
sas_credential_config: $(BLOBFUSE2_SAS_CFG)
|
||||
spn_credential_config: $(BLOBFUSE2_SPN_CFG)
|
||||
distro_name: $(imageName)
|
||||
quick_test: false
|
||||
|
||||
- template: 'azure-pipeline-templates/verbose-tests.yml'
|
||||
parameters:
|
||||
|
|
@ -202,6 +203,7 @@ jobs:
|
|||
sas_credential_config: $(BLOBFUSE2_SAS_CFG)
|
||||
spn_credential_config: $(BLOBFUSE2_SPN_CFG)
|
||||
distro_name: $(imageName)
|
||||
quick_test: false
|
||||
|
||||
- template: azure-pipeline-templates/cleanup.yml
|
||||
parameters:
|
||||
|
|
@ -427,8 +429,7 @@ jobs:
|
|||
temp_dir: $(TEMP_DIR)
|
||||
- script: |
|
||||
kill -9 $(pgrep mitmdump)
|
||||
cat proxy_logs.txt
|
||||
displayName: 'Kill Proxy & Print Proxy Logs'
|
||||
displayName: 'Kill Proxy'
|
||||
|
||||
# End of Ubuntu tests
|
||||
# ----------------------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -10,6 +10,8 @@ import (
|
|||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
|
|
@ -19,6 +21,10 @@ import (
|
|||
var dataValidationMntPathPtr string
|
||||
var dataValidationTempPathPtr string
|
||||
var dataValidationAdlsPtr string
|
||||
var quickTest string
|
||||
var distro string
|
||||
|
||||
var minBuff, medBuff, largeBuff, hugeBuff []byte
|
||||
|
||||
type dataValidationTestSuite struct {
|
||||
suite.Suite
|
||||
|
|
@ -26,9 +32,6 @@ type dataValidationTestSuite struct {
|
|||
testLocalPath string
|
||||
testCachePath string
|
||||
adlsTest bool
|
||||
minBuff []byte
|
||||
medBuff []byte
|
||||
hugeBuff []byte
|
||||
}
|
||||
|
||||
func regDataValidationTestFlag(p *string, name string, value string, usage string) {
|
||||
|
|
@ -45,6 +48,8 @@ func initDataValidationFlags() {
|
|||
dataValidationMntPathPtr = getDataValidationTestFlag("mnt-path")
|
||||
dataValidationAdlsPtr = getDataValidationTestFlag("adls")
|
||||
dataValidationTempPathPtr = getDataValidationTestFlag("tmp-path")
|
||||
quickTest = getDataValidationTestFlag("quick-test")
|
||||
distro = getDataValidationTestFlag("distro-name")
|
||||
}
|
||||
|
||||
func getDataValidationTestDirName(n int) string {
|
||||
|
|
@ -65,18 +70,19 @@ func (suite *dataValidationTestSuite) copyToMountDir(localFilePath string, remot
|
|||
// copy to mounted directory
|
||||
cpCmd := exec.Command("cp", localFilePath, remoteFilePath)
|
||||
cliOut, err := cpCmd.Output()
|
||||
fmt.Println(string(cliOut))
|
||||
if len(cliOut) != 0 {
|
||||
fmt.Println(string(cliOut))
|
||||
}
|
||||
suite.Equal(nil, err)
|
||||
|
||||
// delete the cache directory
|
||||
suite.dataValidationTestCleanup([]string{suite.testCachePath})
|
||||
}
|
||||
|
||||
func (suite *dataValidationTestSuite) validateData(localFilePath string, remoteFilePath string) {
|
||||
// compare the local and mounted files
|
||||
diffCmd := exec.Command("diff", localFilePath, remoteFilePath)
|
||||
cliOut, err := diffCmd.Output()
|
||||
fmt.Println(string(cliOut))
|
||||
if len(cliOut) != 0 {
|
||||
fmt.Println(string(cliOut))
|
||||
}
|
||||
suite.Equal(0, len(cliOut))
|
||||
suite.Equal(nil, err)
|
||||
}
|
||||
|
|
@ -95,10 +101,14 @@ func (suite *dataValidationTestSuite) TestSmallFileData() {
|
|||
srcFile.Close()
|
||||
|
||||
// write to file in the local directory
|
||||
err = ioutil.WriteFile(localFilePath, suite.minBuff, 0777)
|
||||
err = ioutil.WriteFile(localFilePath, minBuff, 0777)
|
||||
suite.Equal(nil, err)
|
||||
|
||||
suite.copyToMountDir(localFilePath, remoteFilePath)
|
||||
|
||||
// delete the cache directory
|
||||
suite.dataValidationTestCleanup([]string{suite.testCachePath})
|
||||
|
||||
suite.validateData(localFilePath, remoteFilePath)
|
||||
|
||||
suite.dataValidationTestCleanup([]string{localFilePath, remoteFilePath, suite.testCachePath})
|
||||
|
|
@ -116,10 +126,14 @@ func (suite *dataValidationTestSuite) TestMediumFileData() {
|
|||
srcFile.Close()
|
||||
|
||||
// write to file in the local directory
|
||||
err = ioutil.WriteFile(localFilePath, suite.medBuff, 0777)
|
||||
err = ioutil.WriteFile(localFilePath, medBuff, 0777)
|
||||
suite.Equal(nil, err)
|
||||
|
||||
suite.copyToMountDir(localFilePath, remoteFilePath)
|
||||
|
||||
// delete the cache directory
|
||||
suite.dataValidationTestCleanup([]string{suite.testCachePath})
|
||||
|
||||
suite.validateData(localFilePath, remoteFilePath)
|
||||
|
||||
suite.dataValidationTestCleanup([]string{localFilePath, remoteFilePath, suite.testCachePath})
|
||||
|
|
@ -137,17 +151,21 @@ func (suite *dataValidationTestSuite) TestLargeFileData() {
|
|||
srcFile.Close()
|
||||
|
||||
// write to file in the local directory
|
||||
err = ioutil.WriteFile(localFilePath, suite.hugeBuff, 0777)
|
||||
err = ioutil.WriteFile(localFilePath, largeBuff, 0777)
|
||||
suite.Equal(nil, err)
|
||||
|
||||
suite.copyToMountDir(localFilePath, remoteFilePath)
|
||||
|
||||
// delete the cache directory
|
||||
suite.dataValidationTestCleanup([]string{suite.testCachePath})
|
||||
|
||||
suite.validateData(localFilePath, remoteFilePath)
|
||||
|
||||
suite.dataValidationTestCleanup([]string{localFilePath, remoteFilePath, suite.testCachePath})
|
||||
}
|
||||
|
||||
// negative test case for data validation where the local file is updated
|
||||
func (suite *dataValidationTestSuite) TestFileUpdate() {
|
||||
func (suite *dataValidationTestSuite) TestDataValidationNegative() {
|
||||
fileName := "updated_data.txt"
|
||||
localFilePath := suite.testLocalPath + "/" + fileName
|
||||
remoteFilePath := suite.testMntPath + "/" + fileName
|
||||
|
|
@ -158,12 +176,15 @@ func (suite *dataValidationTestSuite) TestFileUpdate() {
|
|||
srcFile.Close()
|
||||
|
||||
// write to file in the local directory
|
||||
err = ioutil.WriteFile(localFilePath, suite.minBuff, 0777)
|
||||
err = ioutil.WriteFile(localFilePath, minBuff, 0777)
|
||||
suite.Equal(nil, err)
|
||||
|
||||
// copy local file to mounted directory
|
||||
suite.copyToMountDir(localFilePath, remoteFilePath)
|
||||
|
||||
// delete the cache directory
|
||||
suite.dataValidationTestCleanup([]string{suite.testCachePath})
|
||||
|
||||
// update local file
|
||||
srcFile, err = os.OpenFile(localFilePath, os.O_APPEND|os.O_WRONLY, 0777)
|
||||
suite.Equal(nil, err)
|
||||
|
|
@ -174,6 +195,7 @@ func (suite *dataValidationTestSuite) TestFileUpdate() {
|
|||
// compare local file and mounted files
|
||||
diffCmd := exec.Command("diff", localFilePath, remoteFilePath)
|
||||
cliOut, err := diffCmd.Output()
|
||||
fmt.Println("Negative test case where files should differ")
|
||||
fmt.Println(string(cliOut))
|
||||
suite.NotEqual(0, len(cliOut))
|
||||
suite.NotEqual(nil, err)
|
||||
|
|
@ -181,13 +203,118 @@ func (suite *dataValidationTestSuite) TestFileUpdate() {
|
|||
suite.dataValidationTestCleanup([]string{localFilePath, remoteFilePath, suite.testCachePath})
|
||||
}
|
||||
|
||||
func validateMultipleFilesData(jobs <-chan int, results chan<- string, fileSize string, suite *dataValidationTestSuite) {
|
||||
for i := range jobs {
|
||||
fileName := fileSize + strconv.Itoa(i) + ".txt"
|
||||
localFilePath := suite.testLocalPath + "/" + fileName
|
||||
remoteFilePath := suite.testMntPath + "/" + fileName
|
||||
fmt.Println("Local file path: " + localFilePath)
|
||||
|
||||
// create the file in local directory
|
||||
srcFile, err := os.OpenFile(localFilePath, os.O_CREATE, 0777)
|
||||
suite.Equal(nil, err)
|
||||
srcFile.Close()
|
||||
|
||||
// write to file in the local directory
|
||||
if fileSize == "huge" {
|
||||
err = ioutil.WriteFile(localFilePath, hugeBuff, 0777)
|
||||
} else if fileSize == "large" {
|
||||
if strings.ToLower(quickTest) == "true" {
|
||||
err = ioutil.WriteFile(localFilePath, hugeBuff, 0777)
|
||||
} else {
|
||||
err = ioutil.WriteFile(localFilePath, largeBuff, 0777)
|
||||
}
|
||||
} else if fileSize == "medium" {
|
||||
err = ioutil.WriteFile(localFilePath, medBuff, 0777)
|
||||
} else {
|
||||
err = ioutil.WriteFile(localFilePath, minBuff, 0777)
|
||||
}
|
||||
suite.Equal(nil, err)
|
||||
|
||||
suite.copyToMountDir(localFilePath, remoteFilePath)
|
||||
suite.dataValidationTestCleanup([]string{suite.testCachePath + "/" + fileName})
|
||||
suite.validateData(localFilePath, remoteFilePath)
|
||||
|
||||
suite.dataValidationTestCleanup([]string{localFilePath, suite.testCachePath + "/" + fileName})
|
||||
|
||||
results <- remoteFilePath
|
||||
}
|
||||
}
|
||||
|
||||
func createThreadPool(noOfFiles int, noOfWorkers int, fileSize string, suite *dataValidationTestSuite) {
|
||||
jobs := make(chan int, noOfFiles)
|
||||
results := make(chan string, noOfFiles)
|
||||
|
||||
for i := 1; i <= noOfWorkers; i++ {
|
||||
go validateMultipleFilesData(jobs, results, fileSize, suite)
|
||||
}
|
||||
|
||||
for i := 1; i <= noOfFiles; i++ {
|
||||
jobs <- i
|
||||
}
|
||||
close(jobs)
|
||||
|
||||
for i := 1; i <= noOfFiles; i++ {
|
||||
filePath := <-results
|
||||
os.Remove(filePath)
|
||||
}
|
||||
close(results)
|
||||
|
||||
suite.dataValidationTestCleanup([]string{suite.testCachePath})
|
||||
}
|
||||
|
||||
func (suite *dataValidationTestSuite) TestMultipleSmallFiles() {
|
||||
noOfFiles := 16
|
||||
noOfWorkers := 4
|
||||
createThreadPool(noOfFiles, noOfWorkers, "small", suite)
|
||||
}
|
||||
|
||||
func (suite *dataValidationTestSuite) TestMultipleMediumFiles() {
|
||||
if strings.Contains(strings.ToUpper(distro), "RHEL") {
|
||||
fmt.Println("Skipping this test case for RHEL")
|
||||
return
|
||||
}
|
||||
|
||||
noOfFiles := 8
|
||||
noOfWorkers := 4
|
||||
createThreadPool(noOfFiles, noOfWorkers, "medium", suite)
|
||||
}
|
||||
|
||||
func (suite *dataValidationTestSuite) TestMultipleLargeFiles() {
|
||||
if strings.Contains(strings.ToUpper(distro), "RHEL") {
|
||||
fmt.Println("Skipping this test case for RHEL")
|
||||
return
|
||||
}
|
||||
|
||||
noOfFiles := 4
|
||||
noOfWorkers := 2
|
||||
createThreadPool(noOfFiles, noOfWorkers, "large", suite)
|
||||
}
|
||||
|
||||
func (suite *dataValidationTestSuite) TestMultipleHugeFiles() {
|
||||
if strings.ToLower(quickTest) == "true" {
|
||||
fmt.Println("Quick test is enabled. Skipping this test case")
|
||||
return
|
||||
}
|
||||
|
||||
noOfFiles := 2
|
||||
noOfWorkers := 2
|
||||
createThreadPool(noOfFiles, noOfWorkers, "huge", suite)
|
||||
}
|
||||
|
||||
// -------------- Main Method -------------------
|
||||
func TestDataValidationTestSuite(t *testing.T) {
|
||||
initDataValidationFlags()
|
||||
dataValidationTest := dataValidationTestSuite{
|
||||
minBuff: make([]byte, 1024),
|
||||
medBuff: make([]byte, (10 * 1024 * 1024)),
|
||||
hugeBuff: make([]byte, (500 * 1024 * 1024)),
|
||||
fmt.Println("Distro Name: " + distro)
|
||||
dataValidationTest := dataValidationTestSuite{}
|
||||
|
||||
minBuff = make([]byte, 1024)
|
||||
medBuff = make([]byte, (10 * 1024 * 1024))
|
||||
largeBuff = make([]byte, (500 * 1024 * 1024))
|
||||
if strings.ToLower(quickTest) == "true" {
|
||||
hugeBuff = make([]byte, (100 * 1024 * 1024))
|
||||
} else {
|
||||
hugeBuff = make([]byte, (750 * 1024 * 1024))
|
||||
}
|
||||
|
||||
// Generate random test dir name where our End to End test run is contained
|
||||
|
|
@ -223,9 +350,10 @@ func TestDataValidationTestSuite(t *testing.T) {
|
|||
if err != nil {
|
||||
t.Error("Failed to create test directory")
|
||||
}
|
||||
rand.Read(dataValidationTest.minBuff)
|
||||
rand.Read(dataValidationTest.medBuff)
|
||||
rand.Read(dataValidationTest.hugeBuff)
|
||||
rand.Read(minBuff)
|
||||
rand.Read(medBuff)
|
||||
rand.Read(largeBuff)
|
||||
rand.Read(hugeBuff)
|
||||
|
||||
// Run the actual End to End test
|
||||
suite.Run(t, &dataValidationTest)
|
||||
|
|
@ -238,4 +366,6 @@ func init() {
|
|||
regDataValidationTestFlag(&dataValidationMntPathPtr, "mnt-path", "", "Mount Path of Container")
|
||||
regDataValidationTestFlag(&dataValidationAdlsPtr, "adls", "", "Account is ADLS or not")
|
||||
regDataValidationTestFlag(&dataValidationTempPathPtr, "tmp-path", "", "Cache dir path")
|
||||
regDataValidationTestFlag(&quickTest, "quick-test", "true", "Run quick tests")
|
||||
regDataValidationTestFlag(&distro, "distro-name", "", "Name of the distro")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -24,13 +24,6 @@ file_cache:
|
|||
attr_cache:
|
||||
timeout-sec: 3600
|
||||
|
||||
file_cache:
|
||||
path: { 1 }
|
||||
timeout-sec: 30
|
||||
max-size-mb: 2048
|
||||
allow-non-empty-temp: true
|
||||
cleanup-on-start: true
|
||||
|
||||
azstorage:
|
||||
type: { ACCOUNT_TYPE }
|
||||
endpoint: { ACCOUNT_ENDPOINT }
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче