зеркало из https://github.com/github/vitess-gh.git
453 строки
12 KiB
Go
453 строки
12 KiB
Go
/*
|
|
Copyright 2021 The Vitess Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"fmt"
|
|
"log"
|
|
"os"
|
|
"path"
|
|
"strings"
|
|
"text/template"
|
|
)
|
|
|
|
type mysqlVersion string
|
|
|
|
const (
|
|
mysql57 mysqlVersion = "mysql57"
|
|
mysql80 mysqlVersion = "mysql80"
|
|
mariadb103 mysqlVersion = "mariadb103"
|
|
|
|
defaultMySQLVersion mysqlVersion = mysql57
|
|
)
|
|
|
|
type mysqlVersions []mysqlVersion
|
|
|
|
var (
|
|
defaultMySQLVersions = []mysqlVersion{defaultMySQLVersion}
|
|
mysql80OnlyVersions = []mysqlVersion{mysql80}
|
|
allMySQLVersions = []mysqlVersion{mysql57, mysql80}
|
|
)
|
|
|
|
var (
|
|
unitTestDatabases = []mysqlVersion{mysql57, mysql80, mariadb103}
|
|
)
|
|
|
|
const (
|
|
workflowConfigDir = "../.github/workflows"
|
|
|
|
unitTestTemplate = "templates/unit_test.tpl"
|
|
|
|
// An empty string will cause the default non platform specific template
|
|
// to be used.
|
|
clusterTestTemplate = "templates/cluster_endtoend_test%s.tpl"
|
|
|
|
unitTestSelfHostedTemplate = "templates/unit_test_self_hosted.tpl"
|
|
unitTestSelfHostedDatabases = ""
|
|
dockerFileTemplate = "templates/dockerfile.tpl"
|
|
clusterTestSelfHostedTemplate = "templates/cluster_endtoend_test_self_hosted.tpl"
|
|
clusterTestDockerTemplate = "templates/cluster_endtoend_test_docker.tpl"
|
|
)
|
|
|
|
var (
|
|
// Clusters 10, 25 are executed on docker, using the docker_test_cluster 10, 25 workflows.
|
|
// Hence, they are not listed in the list below.
|
|
clusterList = []string{
|
|
"vtctlbackup_sharded_clustertest_heavy",
|
|
"12",
|
|
"13",
|
|
"ers_prs_newfeatures_heavy",
|
|
"15",
|
|
"vtgate_general_heavy",
|
|
"vtbackup_transform",
|
|
"18",
|
|
"xb_backup",
|
|
"21",
|
|
"22",
|
|
"mysql_server_vault",
|
|
"26",
|
|
"vstream_failover",
|
|
"vstream_stoponreshard_true",
|
|
"vstream_stoponreshard_false",
|
|
"vstream_with_keyspaces_to_watch",
|
|
"onlineddl_ghost",
|
|
"onlineddl_vrepl",
|
|
"onlineddl_vrepl_stress",
|
|
"onlineddl_vrepl_stress_suite",
|
|
"onlineddl_vrepl_suite",
|
|
"vreplication_migrate_vdiff2_convert_tz",
|
|
"onlineddl_revert",
|
|
"onlineddl_declarative",
|
|
"onlineddl_singleton",
|
|
"onlineddl_scheduler",
|
|
"onlineddl_revertible",
|
|
"tabletmanager_throttler",
|
|
"tabletmanager_throttler_custom_config",
|
|
"tabletmanager_tablegc",
|
|
"tabletmanager_consul",
|
|
"vtgate_concurrentdml",
|
|
"vtgate_godriver",
|
|
"vtgate_gen4",
|
|
"vtgate_readafterwrite",
|
|
"vtgate_reservedconn",
|
|
"vtgate_schema",
|
|
"vtgate_tablet_healthcheck_cache",
|
|
"vtgate_topo",
|
|
"vtgate_topo_consul",
|
|
"vtgate_topo_etcd",
|
|
"vtgate_transaction",
|
|
"vtgate_unsharded",
|
|
"vtgate_vindex_heavy",
|
|
"vtgate_vschema",
|
|
"vtgate_queries",
|
|
"vtgate_schema_tracker",
|
|
"xb_recovery",
|
|
"mysql80",
|
|
"vreplication_across_db_versions",
|
|
"vreplication_multicell",
|
|
"vreplication_cellalias",
|
|
"vreplication_basic",
|
|
"vreplication_v2",
|
|
"schemadiff_vrepl",
|
|
"topo_connection_cache",
|
|
"vtgate_partial_keyspace",
|
|
}
|
|
|
|
clusterSelfHostedList = []string{
|
|
"vtorc",
|
|
"vtorc_8.0",
|
|
}
|
|
clusterDockerList = []string{}
|
|
clustersRequiringXtraBackup = []string{
|
|
"xb_backup",
|
|
"xb_recovery",
|
|
}
|
|
clustersRequiringMakeTools = []string{
|
|
"18",
|
|
"mysql_server_vault",
|
|
"vtgate_topo_consul",
|
|
"tabletmanager_consul",
|
|
}
|
|
)
|
|
|
|
type unitTest struct {
|
|
Name, Platform, FileName string
|
|
}
|
|
|
|
type clusterTest struct {
|
|
Name, Shard, Platform string
|
|
FileName string
|
|
MakeTools, InstallXtraBackup bool
|
|
Ubuntu20, Docker bool
|
|
LimitResourceUsage bool
|
|
PartialKeyspace bool
|
|
}
|
|
|
|
type selfHostedTest struct {
|
|
Name, Platform, Dockerfile, Shard, ImageName, directoryName string
|
|
FileName string
|
|
MakeTools, InstallXtraBackup, Docker bool
|
|
}
|
|
|
|
func needsUbuntu20(clusterName string, mysqlVersion mysqlVersion) bool {
|
|
return mysqlVersion == mysql80 || strings.HasPrefix(clusterName, "vtgate") || strings.HasPrefix(clusterName, "tabletmanager")
|
|
}
|
|
|
|
// clusterMySQLVersions return list of mysql versions (one or more) that this cluster needs to test against
|
|
func clusterMySQLVersions(clusterName string) mysqlVersions {
|
|
switch {
|
|
case strings.HasPrefix(clusterName, "onlineddl_"):
|
|
return allMySQLVersions
|
|
case clusterName == "schemadiff_vrepl":
|
|
return allMySQLVersions
|
|
case clusterName == "tabletmanager_tablegc":
|
|
return allMySQLVersions
|
|
case clusterName == "mysql80":
|
|
return mysql80OnlyVersions
|
|
case clusterName == "vtorc_8.0":
|
|
return mysql80OnlyVersions
|
|
case clusterName == "vreplication_across_db_versions":
|
|
return []mysqlVersion{mysql80}
|
|
case clusterName == "xb_backup":
|
|
return allMySQLVersions
|
|
case clusterName == "vtctlbackup_sharded_clustertest_heavy":
|
|
return []mysqlVersion{mysql80}
|
|
case clusterName == "vtbackup_transform":
|
|
return []mysqlVersion{mysql80}
|
|
case clusterName == "vtgate_partial_keyspace":
|
|
return []mysqlVersion{mysql80}
|
|
default:
|
|
return defaultMySQLVersions
|
|
}
|
|
}
|
|
|
|
func mergeBlankLines(buf *bytes.Buffer) string {
|
|
var out []string
|
|
in := strings.Split(buf.String(), "\n")
|
|
lastWasBlank := false
|
|
for _, line := range in {
|
|
if strings.TrimSpace(line) == "" {
|
|
if lastWasBlank {
|
|
continue
|
|
}
|
|
lastWasBlank = true
|
|
} else {
|
|
lastWasBlank = false
|
|
}
|
|
|
|
out = append(out, line)
|
|
}
|
|
return strings.Join(out, "\n")
|
|
}
|
|
|
|
func main() {
|
|
generateUnitTestWorkflows()
|
|
generateClusterWorkflows(clusterList, clusterTestTemplate)
|
|
generateClusterWorkflows(clusterDockerList, clusterTestDockerTemplate)
|
|
|
|
// tests that will use self-hosted runners
|
|
err := generateSelfHostedUnitTestWorkflows()
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
err = generateSelfHostedClusterWorkflows()
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func canonnizeList(list []string) []string {
|
|
var output []string
|
|
for _, item := range list {
|
|
if item := strings.TrimSpace(item); item != "" {
|
|
output = append(output, item)
|
|
}
|
|
}
|
|
return output
|
|
}
|
|
|
|
func parseList(csvList string) []string {
|
|
var list []string
|
|
for _, item := range strings.Split(csvList, ",") {
|
|
if item != "" {
|
|
list = append(list, strings.TrimSpace(item))
|
|
}
|
|
}
|
|
return list
|
|
}
|
|
|
|
func generateSelfHostedUnitTestWorkflows() error {
|
|
platforms := parseList(unitTestSelfHostedDatabases)
|
|
for _, platform := range platforms {
|
|
directoryName := fmt.Sprintf("unit_test_%s", platform)
|
|
test := &selfHostedTest{
|
|
Name: fmt.Sprintf("Unit Test (%s)", platform),
|
|
ImageName: fmt.Sprintf("unit_test_%s", platform),
|
|
Platform: platform,
|
|
directoryName: directoryName,
|
|
Dockerfile: fmt.Sprintf("./.github/docker/%s/Dockerfile", directoryName),
|
|
MakeTools: true,
|
|
InstallXtraBackup: false,
|
|
}
|
|
err := setupTestDockerFile(test)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
test.FileName = fmt.Sprintf("unit_test_%s.yml", platform)
|
|
filePath := fmt.Sprintf("%s/%s", workflowConfigDir, test.FileName)
|
|
err = writeFileFromTemplate(unitTestSelfHostedTemplate, filePath, test)
|
|
if err != nil {
|
|
log.Print(err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func generateSelfHostedClusterWorkflows() error {
|
|
clusters := canonnizeList(clusterSelfHostedList)
|
|
for _, cluster := range clusters {
|
|
for _, mysqlVersion := range clusterMySQLVersions(cluster) {
|
|
directoryName := fmt.Sprintf("cluster_test_%s", cluster)
|
|
test := &selfHostedTest{
|
|
Name: fmt.Sprintf("Cluster (%s)", cluster),
|
|
ImageName: fmt.Sprintf("cluster_test_%s", cluster),
|
|
Platform: "mysql57",
|
|
directoryName: directoryName,
|
|
Dockerfile: fmt.Sprintf("./.github/docker/%s/Dockerfile", directoryName),
|
|
Shard: cluster,
|
|
MakeTools: false,
|
|
InstallXtraBackup: false,
|
|
}
|
|
makeToolClusters := canonnizeList(clustersRequiringMakeTools)
|
|
for _, makeToolCluster := range makeToolClusters {
|
|
if makeToolCluster == cluster {
|
|
test.MakeTools = true
|
|
break
|
|
}
|
|
}
|
|
xtraBackupClusters := canonnizeList(clustersRequiringXtraBackup)
|
|
for _, xtraBackupCluster := range xtraBackupClusters {
|
|
if xtraBackupCluster == cluster {
|
|
test.InstallXtraBackup = true
|
|
break
|
|
}
|
|
}
|
|
if mysqlVersion == mysql80 {
|
|
test.Platform = string(mysql80)
|
|
}
|
|
mysqlVersionIndicator := ""
|
|
if mysqlVersion != defaultMySQLVersion && len(clusterMySQLVersions(cluster)) > 1 {
|
|
mysqlVersionIndicator = "_" + string(mysqlVersion)
|
|
}
|
|
|
|
err := setupTestDockerFile(test)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
test.FileName = fmt.Sprintf("cluster_endtoend_%s%s.yml", cluster, mysqlVersionIndicator)
|
|
filePath := fmt.Sprintf("%s/%s", workflowConfigDir, test.FileName)
|
|
err = writeFileFromTemplate(clusterTestSelfHostedTemplate, filePath, test)
|
|
if err != nil {
|
|
log.Print(err)
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func generateClusterWorkflows(list []string, tpl string) {
|
|
clusters := canonnizeList(list)
|
|
for _, cluster := range clusters {
|
|
for _, mysqlVersion := range clusterMySQLVersions(cluster) {
|
|
test := &clusterTest{
|
|
Name: fmt.Sprintf("Cluster (%s)", cluster),
|
|
Shard: cluster,
|
|
}
|
|
makeToolClusters := canonnizeList(clustersRequiringMakeTools)
|
|
for _, makeToolCluster := range makeToolClusters {
|
|
if makeToolCluster == cluster {
|
|
test.MakeTools = true
|
|
break
|
|
}
|
|
}
|
|
xtraBackupClusters := canonnizeList(clustersRequiringXtraBackup)
|
|
for _, xtraBackupCluster := range xtraBackupClusters {
|
|
if xtraBackupCluster == cluster {
|
|
test.InstallXtraBackup = true
|
|
break
|
|
}
|
|
}
|
|
if needsUbuntu20(cluster, mysqlVersion) {
|
|
test.Ubuntu20 = true
|
|
}
|
|
if mysqlVersion == mysql80 {
|
|
test.Platform = string(mysql80)
|
|
}
|
|
if strings.HasPrefix(cluster, "vreplication") || strings.HasSuffix(cluster, "heavy") {
|
|
test.LimitResourceUsage = true
|
|
}
|
|
mysqlVersionIndicator := ""
|
|
if mysqlVersion != defaultMySQLVersion && len(clusterMySQLVersions(cluster)) > 1 {
|
|
mysqlVersionIndicator = "_" + string(mysqlVersion)
|
|
test.Name = test.Name + " " + string(mysqlVersion)
|
|
}
|
|
if strings.Contains(test.Shard, "partial_keyspace") {
|
|
test.PartialKeyspace = true
|
|
}
|
|
|
|
workflowPath := fmt.Sprintf("%s/cluster_endtoend_%s%s.yml", workflowConfigDir, cluster, mysqlVersionIndicator)
|
|
templateFileName := tpl
|
|
if test.Platform != "" {
|
|
templateFileName = fmt.Sprintf(tpl, "_"+test.Platform)
|
|
} else if strings.Contains(templateFileName, "%s") {
|
|
templateFileName = fmt.Sprintf(tpl, "")
|
|
}
|
|
test.FileName = fmt.Sprintf("cluster_endtoend_%s%s.yml", cluster, mysqlVersionIndicator)
|
|
err := writeFileFromTemplate(templateFileName, workflowPath, test)
|
|
if err != nil {
|
|
log.Print(err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func generateUnitTestWorkflows() {
|
|
for _, platform := range unitTestDatabases {
|
|
test := &unitTest{
|
|
Name: fmt.Sprintf("Unit Test (%s)", platform),
|
|
Platform: string(platform),
|
|
}
|
|
test.FileName = fmt.Sprintf("unit_test_%s.yml", platform)
|
|
path := fmt.Sprintf("%s/%s", workflowConfigDir, test.FileName)
|
|
err := writeFileFromTemplate(unitTestTemplate, path, test)
|
|
if err != nil {
|
|
log.Print(err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func setupTestDockerFile(test *selfHostedTest) error {
|
|
// remove the directory
|
|
relDirectoryName := fmt.Sprintf("../.github/docker/%s", test.directoryName)
|
|
err := os.RemoveAll(relDirectoryName)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// create the directory
|
|
err = os.MkdirAll(relDirectoryName, 0755)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// generate the docker file
|
|
dockerFilePath := path.Join(relDirectoryName, "Dockerfile")
|
|
err = writeFileFromTemplate(dockerFileTemplate, dockerFilePath, test)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func writeFileFromTemplate(templateFile, path string, test any) error {
|
|
tpl, err := template.ParseFiles(templateFile)
|
|
if err != nil {
|
|
return fmt.Errorf("Error: %s\n", err)
|
|
}
|
|
|
|
buf := &bytes.Buffer{}
|
|
err = tpl.Execute(buf, test)
|
|
if err != nil {
|
|
return fmt.Errorf("Error: %s\n", err)
|
|
}
|
|
|
|
f, err := os.Create(path)
|
|
if err != nil {
|
|
return fmt.Errorf("Error creating file: %s\n", err)
|
|
}
|
|
if _, err := f.WriteString("# DO NOT MODIFY: THIS FILE IS GENERATED USING \"make generate_ci_workflows\"\n\n"); err != nil {
|
|
return err
|
|
}
|
|
if _, err := f.WriteString(mergeBlankLines(buf)); err != nil {
|
|
return err
|
|
}
|
|
fmt.Printf("Generated %s\n", path)
|
|
return nil
|
|
}
|