E2E data validation tests (#740)
* Adding e2e test for data validation * Correcting tmp-path flag name * Not run as unit test * Adding negative test case
This commit is contained in:
Sourav Gupta
GitHub
Родитель
50c2267008
Коммит
5cda7edeab
|
|
@ -40,7 +40,7 @@ steps:
|
|||
- task: Go@0
|
||||
inputs:
|
||||
command: 'test'
|
||||
arguments: '-v -timeout=2h ./... -args -mnt-path=${{ parameters.mount_dir }} -adls=${{parameters.adls}} -clone=${{parameters.clone}}'
|
||||
arguments: '-v -timeout=2h ./... -args -mnt-path=${{ parameters.mount_dir }} -adls=${{parameters.adls}} -clone=${{parameters.clone}} -tmp-path=${{parameters.temp_dir}}'
|
||||
workingDirectory: ${{ parameters.working_dir }}/test/e2e_tests
|
||||
displayName: 'E2E Test: ${{ parameters.idstring }}'
|
||||
timeoutInMinutes: 120
|
||||
|
|
|
|||
|
|
@ -116,7 +116,7 @@ jobs:
|
|||
sleep 10
|
||||
ps -aux | grep blobfuse2
|
||||
rm -rf $(MOUNT_DIR)/*
|
||||
go test -v -timeout=7200s test/e2e_tests -args -mnt-path=$(MOUNT_DIR)
|
||||
go test -v -timeout=7200s test/e2e_tests -args -mnt-path=$(MOUNT_DIR) -tmp-path=$(TEMP_DIR)
|
||||
sudo fusermount -u $(MOUNT_DIR)
|
||||
sleep 5
|
||||
workingDirectory: $(WORK_DIR)
|
||||
|
|
@ -141,7 +141,7 @@ jobs:
|
|||
sleep 10
|
||||
ps -aux | grep blobfuse2
|
||||
rm -rf $(MOUNT_DIR)/*
|
||||
go test -v -timeout=7200s test/e2e_tests -args -mnt-path=$(MOUNT_DIR) -adls=true
|
||||
go test -v -timeout=7200s test/e2e_tests -args -mnt-path=$(MOUNT_DIR) -adls=true -tmp-path=$(TEMP_DIR)
|
||||
./blobfuse2 unmount all
|
||||
sleep 5
|
||||
workingDirectory: $(WORK_DIR)
|
||||
|
|
@ -169,7 +169,7 @@ jobs:
|
|||
sleep 10
|
||||
ps -aux | grep blobfuse2
|
||||
rm -rf $(MOUNT_DIR)/*
|
||||
go test -v -timeout=7200s test/e2e_tests -args -mnt-path=$(MOUNT_DIR)
|
||||
go test -v -timeout=7200s test/e2e_tests -args -mnt-path=$(MOUNT_DIR) -tmp-path=$(TEMP_DIR)
|
||||
./blobfuse2 unmount $(MOUNT_DIR)
|
||||
sleep 5
|
||||
workingDirectory: $(WORK_DIR)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,241 @@
|
|||
// +build !unittest
|
||||
|
||||
package e2e_tests
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/suite"
|
||||
)
|
||||
|
||||
var dataValidationMntPathPtr string
|
||||
var dataValidationTempPathPtr string
|
||||
var dataValidationAdlsPtr string
|
||||
|
||||
type dataValidationTestSuite struct {
|
||||
suite.Suite
|
||||
testMntPath string
|
||||
testLocalPath string
|
||||
testCachePath string
|
||||
adlsTest bool
|
||||
minBuff []byte
|
||||
medBuff []byte
|
||||
hugeBuff []byte
|
||||
}
|
||||
|
||||
func regDataValidationTestFlag(p *string, name string, value string, usage string) {
|
||||
if flag.Lookup(name) == nil {
|
||||
flag.StringVar(p, name, value, usage)
|
||||
}
|
||||
}
|
||||
|
||||
func getDataValidationTestFlag(name string) string {
|
||||
return flag.Lookup(name).Value.(flag.Getter).Get().(string)
|
||||
}
|
||||
|
||||
func initDataValidationFlags() {
|
||||
dataValidationMntPathPtr = getDataValidationTestFlag("mnt-path")
|
||||
dataValidationAdlsPtr = getDataValidationTestFlag("adls")
|
||||
dataValidationTempPathPtr = getDataValidationTestFlag("tmp-path")
|
||||
}
|
||||
|
||||
func getDataValidationTestDirName(n int) string {
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
b := make([]byte, n)
|
||||
rand.Read(b)
|
||||
return fmt.Sprintf("%x", b)[:n]
|
||||
}
|
||||
|
||||
func (suite *dataValidationTestSuite) dataValidationTestCleanup(toRemove []string) {
|
||||
for _, path := range toRemove {
|
||||
err := os.RemoveAll(path)
|
||||
suite.Equal(nil, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (suite *dataValidationTestSuite) copyToMountDir(localFilePath string, remoteFilePath string) {
|
||||
// copy to mounted directory
|
||||
cpCmd := exec.Command("cp", localFilePath, remoteFilePath)
|
||||
cliOut, err := cpCmd.Output()
|
||||
fmt.Println(string(cliOut))
|
||||
suite.Equal(nil, err)
|
||||
|
||||
// delete the cache directory
|
||||
suite.dataValidationTestCleanup([]string{suite.testCachePath})
|
||||
}
|
||||
|
||||
func (suite *dataValidationTestSuite) validateData(localFilePath string, remoteFilePath string) {
|
||||
// compare the local and mounted files
|
||||
diffCmd := exec.Command("diff", localFilePath, remoteFilePath)
|
||||
cliOut, err := diffCmd.Output()
|
||||
fmt.Println(string(cliOut))
|
||||
suite.Equal(0, len(cliOut))
|
||||
suite.Equal(nil, err)
|
||||
}
|
||||
|
||||
// -------------- Data Validation Tests -------------------
|
||||
|
||||
// data validation for small sized files
|
||||
func (suite *dataValidationTestSuite) TestSmallFileData() {
|
||||
fileName := "small_data.txt"
|
||||
localFilePath := suite.testLocalPath + "/" + fileName
|
||||
remoteFilePath := suite.testMntPath + "/" + fileName
|
||||
|
||||
// create the file in local directory
|
||||
srcFile, err := os.OpenFile(localFilePath, os.O_CREATE, 0777)
|
||||
suite.Equal(nil, err)
|
||||
srcFile.Close()
|
||||
|
||||
// write to file in the local directory
|
||||
err = ioutil.WriteFile(localFilePath, suite.minBuff, 0777)
|
||||
suite.Equal(nil, err)
|
||||
|
||||
suite.copyToMountDir(localFilePath, remoteFilePath)
|
||||
suite.validateData(localFilePath, remoteFilePath)
|
||||
|
||||
suite.dataValidationTestCleanup([]string{localFilePath, remoteFilePath, suite.testCachePath})
|
||||
}
|
||||
|
||||
// data validation for medium sized files
|
||||
func (suite *dataValidationTestSuite) TestMediumFileData() {
|
||||
fileName := "medium_data.txt"
|
||||
localFilePath := suite.testLocalPath + "/" + fileName
|
||||
remoteFilePath := suite.testMntPath + "/" + fileName
|
||||
|
||||
// create the file in local directory
|
||||
srcFile, err := os.OpenFile(localFilePath, os.O_CREATE, 0777)
|
||||
suite.Equal(nil, err)
|
||||
srcFile.Close()
|
||||
|
||||
// write to file in the local directory
|
||||
err = ioutil.WriteFile(localFilePath, suite.medBuff, 0777)
|
||||
suite.Equal(nil, err)
|
||||
|
||||
suite.copyToMountDir(localFilePath, remoteFilePath)
|
||||
suite.validateData(localFilePath, remoteFilePath)
|
||||
|
||||
suite.dataValidationTestCleanup([]string{localFilePath, remoteFilePath, suite.testCachePath})
|
||||
}
|
||||
|
||||
// data validation for large sized files
|
||||
func (suite *dataValidationTestSuite) TestLargeFileData() {
|
||||
fileName := "large_data.txt"
|
||||
localFilePath := suite.testLocalPath + "/" + fileName
|
||||
remoteFilePath := suite.testMntPath + "/" + fileName
|
||||
|
||||
// create the file in local directory
|
||||
srcFile, err := os.OpenFile(localFilePath, os.O_CREATE, 0777)
|
||||
suite.Equal(nil, err)
|
||||
srcFile.Close()
|
||||
|
||||
// write to file in the local directory
|
||||
err = ioutil.WriteFile(localFilePath, suite.hugeBuff, 0777)
|
||||
suite.Equal(nil, err)
|
||||
|
||||
suite.copyToMountDir(localFilePath, remoteFilePath)
|
||||
suite.validateData(localFilePath, remoteFilePath)
|
||||
|
||||
suite.dataValidationTestCleanup([]string{localFilePath, remoteFilePath, suite.testCachePath})
|
||||
}
|
||||
|
||||
// negative test case for data validation where the local file is updated
|
||||
func (suite *dataValidationTestSuite) TestFileUpdate() {
|
||||
fileName := "updated_data.txt"
|
||||
localFilePath := suite.testLocalPath + "/" + fileName
|
||||
remoteFilePath := suite.testMntPath + "/" + fileName
|
||||
|
||||
// create the file in local directory
|
||||
srcFile, err := os.OpenFile(localFilePath, os.O_CREATE, 0777)
|
||||
suite.Equal(nil, err)
|
||||
srcFile.Close()
|
||||
|
||||
// write to file in the local directory
|
||||
err = ioutil.WriteFile(localFilePath, suite.minBuff, 0777)
|
||||
suite.Equal(nil, err)
|
||||
|
||||
// copy local file to mounted directory
|
||||
suite.copyToMountDir(localFilePath, remoteFilePath)
|
||||
|
||||
// update local file
|
||||
srcFile, err = os.OpenFile(localFilePath, os.O_APPEND|os.O_WRONLY, 0777)
|
||||
suite.Equal(nil, err)
|
||||
_, err = srcFile.WriteString("Added text")
|
||||
srcFile.Close()
|
||||
suite.Equal(nil, err)
|
||||
|
||||
// compare local file and mounted files
|
||||
diffCmd := exec.Command("diff", localFilePath, remoteFilePath)
|
||||
cliOut, err := diffCmd.Output()
|
||||
fmt.Println(string(cliOut))
|
||||
suite.NotEqual(0, len(cliOut))
|
||||
suite.NotEqual(nil, err)
|
||||
|
||||
suite.dataValidationTestCleanup([]string{localFilePath, remoteFilePath, suite.testCachePath})
|
||||
}
|
||||
|
||||
// -------------- Main Method -------------------
|
||||
func TestDataValidationTestSuite(t *testing.T) {
|
||||
initDataValidationFlags()
|
||||
dataValidationTest := dataValidationTestSuite{
|
||||
minBuff: make([]byte, 1024),
|
||||
medBuff: make([]byte, (10 * 1024 * 1024)),
|
||||
hugeBuff: make([]byte, (500 * 1024 * 1024)),
|
||||
}
|
||||
|
||||
// Generate random test dir name where our End to End test run is contained
|
||||
testDirName := getDataValidationTestDirName(10)
|
||||
|
||||
// Create directory for testing the End to End test on mount path
|
||||
dataValidationTest.testMntPath = dataValidationMntPathPtr + "/" + testDirName
|
||||
fmt.Println(dataValidationTest.testMntPath)
|
||||
|
||||
dataValidationTest.testLocalPath, _ = filepath.Abs(dataValidationMntPathPtr + "/..")
|
||||
fmt.Println(dataValidationTest.testLocalPath)
|
||||
|
||||
dataValidationTest.testCachePath = dataValidationTempPathPtr + "/" + testDirName
|
||||
fmt.Println(dataValidationTest.testCachePath)
|
||||
|
||||
if dataValidationAdlsPtr == "true" || dataValidationAdlsPtr == "True" {
|
||||
fmt.Println("ADLS Testing...")
|
||||
dataValidationTest.adlsTest = true
|
||||
} else {
|
||||
fmt.Println("BLOCK Blob Testing...")
|
||||
}
|
||||
// Sanity check in the off chance the same random name was generated twice and was still around somehow
|
||||
err := os.RemoveAll(dataValidationTest.testMntPath)
|
||||
if err != nil {
|
||||
fmt.Println("Could not cleanup feature dir before testing")
|
||||
}
|
||||
err = os.RemoveAll(dataValidationTest.testCachePath)
|
||||
if err != nil {
|
||||
fmt.Println("Could not cleanup cache dir before testing")
|
||||
}
|
||||
|
||||
err = os.Mkdir(dataValidationTest.testMntPath, 0777)
|
||||
if err != nil {
|
||||
t.Error("Failed to create test directory")
|
||||
}
|
||||
rand.Read(dataValidationTest.minBuff)
|
||||
rand.Read(dataValidationTest.medBuff)
|
||||
rand.Read(dataValidationTest.hugeBuff)
|
||||
|
||||
// Run the actual End to End test
|
||||
suite.Run(t, &dataValidationTest)
|
||||
|
||||
// Wipe out the test directory created for End to End test
|
||||
os.RemoveAll(dataValidationTest.testMntPath)
|
||||
}
|
||||
|
||||
func init() {
|
||||
regDataValidationTestFlag(&dataValidationMntPathPtr, "mnt-path", "", "Mount Path of Container")
|
||||
regDataValidationTestFlag(&dataValidationAdlsPtr, "adls", "", "Account is ADLS or not")
|
||||
regDataValidationTestFlag(&dataValidationTempPathPtr, "tmp-path", "", "Cache dir path")
|
||||
}
|
||||
Загрузка…
Ссылка в новой задаче