added go utility to run concurrent deployment/validation tests
This commit is contained in:
Dmitry Shmulevich
Родитель
ce3c894b17
Коммит
91e9569426
1
Makefile
1
Makefile
|
|
@ -9,6 +9,7 @@ build: prereqs
|
|||
go generate -v ./...
|
||||
go get .
|
||||
go build -v
|
||||
cd test/acs-engine-test; go build -v
|
||||
|
||||
test: prereqs
|
||||
go test -v ./...
|
||||
|
|
|
|||
|
|
@ -0,0 +1,30 @@
|
|||
#!/bin/bash
|
||||
|
||||
SOURCE="${BASH_SOURCE[0]}"
|
||||
while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
|
||||
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
|
||||
SOURCE="$(readlink "$SOURCE")"
|
||||
[[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
|
||||
done
|
||||
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
|
||||
|
||||
###############################################################################
|
||||
|
||||
set -e
|
||||
set -u
|
||||
set -o pipefail
|
||||
|
||||
ROOT="${DIR}/.."
|
||||
|
||||
# Check pre-requisites
|
||||
[[ ! -z "${SERVICE_PRINCIPAL_CLIENT_ID:-}" ]] || (echo "Must specify SERVICE_PRINCIPAL_CLIENT_ID" && exit -1)
|
||||
[[ ! -z "${SERVICE_PRINCIPAL_CLIENT_SECRET:-}" ]] || (echo "Must specify SERVICE_PRINCIPAL_CLIENT_SECRET" && exit -1)
|
||||
[[ ! -z "${TENANT_ID:-}" ]] || (echo "Must specify TENANT_ID" && exit -1)
|
||||
[[ ! -z "${SUBSCRIPTION_ID:-}" ]] || (echo "Must specify SUBSCRIPTION_ID" && exit -1)
|
||||
[[ ! -z "${CLUSTER_SERVICE_PRINCIPAL_CLIENT_ID:-}" ]] || (echo "Must specify CLUSTER_SERVICE_PRINCIPAL_CLIENT_ID" && exit -1)
|
||||
[[ ! -z "${CLUSTER_SERVICE_PRINCIPAL_CLIENT_SECRET:-}" ]] || (echo "Must specify CLUSTER_SERVICE_PRINCIPAL_CLIENT_SECRET" && exit -1)
|
||||
[[ ! -z "${STAGE_TIMEOUT_MIN:-}" ]] || (echo "Must specify STAGE_TIMEOUT_MIN" && exit -1)
|
||||
|
||||
make -C "${ROOT}" ci
|
||||
|
||||
${ROOT}/test/acs-engine-test/acs-engine-test -c ${TEST_CONFIG:-${ROOT}/test/acs-engine-test/acs-engine-test.json} -d ${ROOT}
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
{"deployments":
|
||||
[
|
||||
{
|
||||
"cluster_definition":"examples/kubernetes.json",
|
||||
"location":"westus"
|
||||
},
|
||||
{
|
||||
"cluster_definition":"examples/dcos.json",
|
||||
"location":"eastus"
|
||||
},
|
||||
{
|
||||
"cluster_definition":"examples/swarm.json",
|
||||
"location":"southcentralus"
|
||||
},
|
||||
{
|
||||
"cluster_definition":"examples/swarmmode.json",
|
||||
"location":"westus2"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io/ioutil"
|
||||
)
|
||||
|
||||
type Deployment struct {
|
||||
ClusterDefinition string `json:"cluster_definition"`
|
||||
Location string `json:"location"`
|
||||
}
|
||||
|
||||
type TestConfig struct {
|
||||
Deployments []Deployment `json:"deployments"`
|
||||
}
|
||||
|
||||
func (c *TestConfig) Read(data []byte) error {
|
||||
return json.Unmarshal(data, c)
|
||||
}
|
||||
|
||||
func (c *TestConfig) Validate() error {
|
||||
for _, d := range c.Deployments {
|
||||
if d.ClusterDefinition == "" {
|
||||
return errors.New("Cluster definition is not set")
|
||||
}
|
||||
if d.Location == "" {
|
||||
return errors.New("Location is not set")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func getTestConfig(fname string) (*TestConfig, error) {
|
||||
data, err := ioutil.ReadFile(fname)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
config := new(TestConfig)
|
||||
if err = config.Read(data); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err = config.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return config, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
package main
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestConfigParse(t *testing.T) {
|
||||
|
||||
test_cfg := `
|
||||
{"deployments":
|
||||
[
|
||||
{
|
||||
"cluster_definition":"examples/kubernetes.json",
|
||||
"location":"westus"
|
||||
},
|
||||
{
|
||||
"cluster_definition":"examples/dcos.json",
|
||||
"location":"eastus"
|
||||
},
|
||||
{
|
||||
"cluster_definition":"examples/swarm.json",
|
||||
"location":"southcentralus"
|
||||
},
|
||||
{
|
||||
"cluster_definition":"examples/swarmmode.json",
|
||||
"location":"westus2"
|
||||
}
|
||||
]
|
||||
}
|
||||
`
|
||||
|
||||
testConfig := TestConfig{}
|
||||
if err := testConfig.Read([]byte(test_cfg)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := testConfig.Validate(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(testConfig.Deployments) != 4 {
|
||||
t.Fatalf("Wrong number of deployments: %d instead of 4", len(testConfig.Deployments))
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,224 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const script = "test/step.sh"
|
||||
|
||||
const usage = `Usage:
|
||||
acs-engine-test -c <configuration.json> -d <acs-engine root directory>
|
||||
|
||||
Options:
|
||||
-c <configuration.json> : JSON file containing a list of deployment configurations.
|
||||
Refer to acs-engine/test/acs-engine-test/acs-engine-test.json for examples
|
||||
-d <acs-engine root directory>
|
||||
`
|
||||
|
||||
var logDir string
|
||||
var orchestrator_re *regexp.Regexp
|
||||
|
||||
func init() {
|
||||
orchestrator_re = regexp.MustCompile(`"orchestratorType": "(\S+)"`)
|
||||
}
|
||||
|
||||
type TestManager struct {
|
||||
config *TestConfig
|
||||
lock sync.Mutex
|
||||
wg sync.WaitGroup
|
||||
rootDir string
|
||||
}
|
||||
|
||||
func (m *TestManager) Run() error {
|
||||
n := len(m.config.Deployments)
|
||||
if n == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// deternime timeout
|
||||
timeoutMin, err := strconv.Atoi(os.Getenv("STAGE_TIMEOUT_MIN"))
|
||||
if err != nil {
|
||||
fmt.Printf("Error [Atoi STAGE_TIMEOUT_MIN]: %v\n", err)
|
||||
return err
|
||||
}
|
||||
timeout := time.Duration(time.Minute * time.Duration(timeoutMin))
|
||||
|
||||
// login to Azure
|
||||
if err := runStep("set_azure_account", m.rootDir, "main", os.Environ(), fmt.Sprintf("%s/main.log", logDir), timeout); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// return values for tests
|
||||
retvals := make([]byte, n)
|
||||
|
||||
m.wg.Add(n)
|
||||
for i, d := range m.config.Deployments {
|
||||
go func(i int, d Deployment) {
|
||||
defer m.wg.Done()
|
||||
|
||||
name := strings.TrimSuffix(d.ClusterDefinition, filepath.Ext(d.ClusterDefinition))
|
||||
instanceName := fmt.Sprintf("test-acs-%s-%s-%s-%d", strings.Replace(name, "/", "-", -1), d.Location, os.Getenv("BUILD_NUMBER"), i)
|
||||
logFile := fmt.Sprintf("%s/%s.log", logDir, instanceName)
|
||||
|
||||
// determine orchestrator
|
||||
orchestrator, err := getOrchestrator(fmt.Sprintf("%s/%s", m.rootDir, d.ClusterDefinition))
|
||||
if err != nil {
|
||||
wrileLog(logFile, []byte(err.Error()))
|
||||
fmt.Printf("Error [getOrchestrator %s] : %v\n", d.ClusterDefinition, err)
|
||||
retvals[i] = 1
|
||||
return
|
||||
}
|
||||
|
||||
// update environment
|
||||
env := os.Environ()
|
||||
env = append(env, fmt.Sprintf("CLUSTER_DEFINITION=%s", d.ClusterDefinition))
|
||||
env = append(env, fmt.Sprintf("LOCATION=%s", d.Location))
|
||||
env = append(env, fmt.Sprintf("ORCHESTRATOR=%s", orchestrator))
|
||||
env = append(env, fmt.Sprintf("INSTANCE_NAME=%s", instanceName))
|
||||
env = append(env, fmt.Sprintf("DEPLOYMENT_NAME=%s", instanceName))
|
||||
env = append(env, fmt.Sprintf("RESOURCE_GROUP=%s", instanceName))
|
||||
|
||||
steps := []string{"generate_template", "deploy_template"}
|
||||
|
||||
// determine validation script
|
||||
validate := fmt.Sprintf("test/cluster-tests/%s/test.sh", orchestrator)
|
||||
if _, err = os.Stat(fmt.Sprintf("%s/%s", m.rootDir, validate)); err == nil {
|
||||
env = append(env, fmt.Sprintf("VALIDATE=%s", validate))
|
||||
steps = append(steps, "validate")
|
||||
}
|
||||
|
||||
for _, step := range steps {
|
||||
if err = runStep(step, m.rootDir, instanceName, env, logFile, timeout); err != nil {
|
||||
retvals[i] = 1
|
||||
break
|
||||
}
|
||||
}
|
||||
// clean up
|
||||
runStep("cleanup", m.rootDir, instanceName, env, logFile, timeout)
|
||||
}(i, d)
|
||||
}
|
||||
m.wg.Wait()
|
||||
for _, retval := range retvals {
|
||||
if retval != 0 {
|
||||
return errors.New("Test failed")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func getOrchestrator(fname string) (string, error) {
|
||||
data, err := ioutil.ReadFile(fname)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
parts := orchestrator_re.FindStringSubmatch(line)
|
||||
if parts != nil {
|
||||
orchestrator := strings.ToLower(parts[1])
|
||||
if strings.HasPrefix(orchestrator, "dcos") {
|
||||
orchestrator = "dcos"
|
||||
}
|
||||
return orchestrator, nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("No orchestratorType in %s", fname)
|
||||
}
|
||||
|
||||
func runStep(step, dir, instanceName string, env []string, logFile string, timeout time.Duration) error {
|
||||
cmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("%s %s", script, step))
|
||||
cmd.Dir = dir
|
||||
cmd.Env = env
|
||||
|
||||
var out bytes.Buffer
|
||||
cmd.Stdout = &out
|
||||
cmd.Stderr = &out
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
fmt.Printf("Error [%s %s] : %v\n", step, instanceName, err)
|
||||
return err
|
||||
}
|
||||
timer := time.AfterFunc(timeout, func() {
|
||||
cmd.Process.Kill()
|
||||
})
|
||||
err := cmd.Wait()
|
||||
timer.Stop()
|
||||
|
||||
wrileLog(logFile, out.Bytes())
|
||||
if err != nil {
|
||||
fmt.Printf("Error [%s %s] : %v\n", step, instanceName, err)
|
||||
return err
|
||||
}
|
||||
fmt.Printf("SUCCESS [%s %s]\n", step, instanceName)
|
||||
return nil
|
||||
}
|
||||
|
||||
func wrileLog(fname string, data []byte) {
|
||||
f, err := os.OpenFile(fname, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
|
||||
if err != nil {
|
||||
fmt.Printf("Error [OpenFile %s] : %v\n", fname, err)
|
||||
return
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
if _, err = f.Write(data); err != nil {
|
||||
fmt.Printf("Error [Write %s] : %v\n", fname, err)
|
||||
}
|
||||
}
|
||||
|
||||
func main_internal() error {
|
||||
var configFile string
|
||||
var rootDir string
|
||||
var err error
|
||||
flag.StringVar(&configFile, "c", "", "deployment configurations")
|
||||
flag.StringVar(&rootDir, "d", "", "acs-engine root directory")
|
||||
flag.Usage = func() {
|
||||
fmt.Println(usage)
|
||||
}
|
||||
flag.Parse()
|
||||
|
||||
testManager := TestManager{}
|
||||
|
||||
// get test configuration
|
||||
if configFile == "" {
|
||||
return fmt.Errorf("test configuration is not provided")
|
||||
}
|
||||
testManager.config, err = getTestConfig(configFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// check root directory
|
||||
if rootDir == "" {
|
||||
return fmt.Errorf("acs-engine root directory is not provided")
|
||||
}
|
||||
testManager.rootDir = rootDir
|
||||
if _, err = os.Stat(fmt.Sprintf("%s/%s", rootDir, script)); err != nil {
|
||||
return err
|
||||
}
|
||||
// make logs directory
|
||||
logDir = fmt.Sprintf("%s/_logs", rootDir)
|
||||
if err = os.Mkdir(logDir, os.FileMode(0755)); err != nil {
|
||||
return err
|
||||
}
|
||||
// run tests
|
||||
return testManager.Run()
|
||||
}
|
||||
|
||||
func main() {
|
||||
if err := main_internal(); err != nil {
|
||||
fmt.Printf("Error: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
os.Exit(0)
|
||||
}
|
||||
11
test/step.sh
11
test/step.sh
|
|
@ -16,21 +16,26 @@ set -o pipefail
|
|||
|
||||
ROOT="${DIR}/.."
|
||||
|
||||
# Set output directory
|
||||
export OUTPUT="${ROOT}/_output/${INSTANCE_NAME}"
|
||||
|
||||
source "${ROOT}/test/common.sh"
|
||||
|
||||
case $1 in
|
||||
|
||||
set_azure_account)
|
||||
set_azure_account
|
||||
;;
|
||||
|
||||
generate_template)
|
||||
export OUTPUT="${ROOT}/_output/${INSTANCE_NAME}"
|
||||
generate_template
|
||||
;;
|
||||
|
||||
deploy_template)
|
||||
export OUTPUT="${ROOT}/_output/${INSTANCE_NAME}"
|
||||
deploy_template
|
||||
;;
|
||||
|
||||
validate)
|
||||
export OUTPUT="${ROOT}/_output/${INSTANCE_NAME}"
|
||||
export SSH_KEY="${OUTPUT}/id_rsa"
|
||||
if [ ${ORCHESTRATOR} = "kubernetes" ]; then
|
||||
export KUBECONFIG="${OUTPUT}/kubeconfig/kubeconfig.${LOCATION}.json"
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче