test.go: Add option to run only specified tests.

test.go

@@ -32,11 +32,19 @@ import (
 	"os/exec"
 	"os/signal"
 	"path"
-	"strings"
+	"sort"
 	"syscall"
 	"time"
 )
 
+var usage = `Usage of test.go:
+
+go run test.go [options] [test_name ...]
+
+If one or more test names are provided, run only those tests.
+Otherwise, run all tests in test/config.json.
+`
+
 // Flags
 var (
 	flavor   = flag.String("flavor", "mariadb", "bootstrap flavor to run against")
@@ -49,7 +57,7 @@ var (
 
 // Config is the overall object serialized in test/config.json.
 type Config struct {
-	Tests []*Test
+	Tests map[string]*Test
 }
 
 // Test is an entry from the test/config.json file.
@@ -120,8 +128,15 @@ func (t *Test) logf(format string, v ...interface{}) {
 }
 
 func main() {
+	flag.Usage = func() {
+		os.Stderr.WriteString(usage)
+		os.Stderr.WriteString("\nOptions:\n")
+		flag.PrintDefaults()
+	}
 	flag.Parse()
 
+	startTime := time.Now()
+
 	// Get test configs.
 	configData, err := ioutil.ReadFile("test/config.json")
 	if err != nil {
@@ -133,6 +148,31 @@ func main() {
 	}
 	log.Printf("Bootstrap flavor: %v", *flavor)
 
+	// Positional args specify which tests to run.
+	// If none specified, run all tests in alphabetical order.
+	var tests []*Test
+	if flag.NArg() > 0 {
+		for _, name := range flag.Args() {
+			t, ok := config.Tests[name]
+			if !ok {
+				log.Fatalf("Unknown test: %v", name)
+			}
+			t.Name = name
+			tests = append(tests, t)
+		}
+	} else {
+		names := make([]string, 0, len(config.Tests))
+		for n := range config.Tests {
+			names = append(names, n)
+		}
+		sort.Strings(names)
+		for _, n := range names {
+			t := config.Tests[n]
+			t.Name = n
+			tests = append(tests, t)
+		}
+	}
+
 	// Copy working repo to tmpDir.
 	tmpDir, err := ioutil.TempDir(os.TempDir(), "vt_")
 	if err != nil {
@@ -165,11 +205,7 @@ func main() {
 			close(done)
 		}()
 
-		for _, test := range config.Tests {
-			if test.Name == "" {
-				test.Name = strings.TrimSuffix(test.File, ".py")
-			}
-
+		for _, test := range tests {
 			for try := 1; ; try++ {
 				select {
 				case <-stop:
@@ -189,7 +225,7 @@ func main() {
 				start := time.Now()
 				if err := test.run(tmpDir); err != nil {
 					// This try failed.
-					test.logf("FAILED (try %v/%v): %v", try, *retryMax, err)
+					test.logf("FAILED (try %v/%v) in %v: %v", try, *retryMax, time.Since(start), err)
 					continue
 				}
 
@@ -199,7 +235,7 @@ func main() {
 					passed++
 				} else {
 					// Passed, but not on the first try.
-					test.logf("FLAKY (1/%v passed)", try)
+					test.logf("FLAKY (1/%v passed in %v)", try, time.Since(start))
 					flaky++
 				}
 				break
@@ -228,8 +264,9 @@ func main() {
 	}
 
 	// Print stats.
-	skipped := len(config.Tests) - passed - flaky - failed
+	skipped := len(tests) - passed - flaky - failed
 	log.Printf("%v PASSED, %v FLAKY, %v FAILED, %v SKIPPED", passed, flaky, failed, skipped)
+	log.Printf("Total time: %v", time.Since(startTime))
 
 	if failed > 0 || skipped > 0 {
 		os.Exit(1)

test/config.json

@@ -1,86 +1,84 @@
 {
-  "Tests": [
-    {
-      "Name": "queryservice_vtocc",
+  "Tests": {
+    "queryservice_vtocc": {
       "File": "queryservice_test.py",
       "Args": "-m -e vtocc"
     },
-    {
-      "Name": "queryservice_vttablet",
+    "queryservice_vttablet": {
       "File": "queryservice_test.py",
       "Args": "-m -e vttablet"
     },
-    {
+    "vertical_split": {
       "File": "vertical_split.py"
     },
-    {
+    "vertical_split_vtgate": {
       "File": "vertical_split_vtgate.py"
     },
-    {
+    "schema": {
       "File": "schema.py"
     },
-    {
+    "keyspace": {
       "File": "keyspace_test.py"
     },
-    {
+    "keyrange": {
       "File": "keyrange_test.py"
     },
-    {
+    "mysqlctl": {
       "File": "mysqlctl.py"
     },
-    {
+    "sharded": {
       "File": "sharded.py"
     },
-    {
+    "secure": {
       "File": "secure.py"
     },
-    {
+    "binlog": {
       "File": "binlog.py"
     },
-    {
+    "backup": {
       "File": "backup.py"
     },
-    {
+    "custom_sharding": {
       "File": "custom_sharding.py"
     },
-    {
+    "update_stream": {
       "File": "update_stream.py"
     },
-    {
+    "tabletmanager": {
       "File": "tabletmanager.py"
     },
-    {
+    "reparent": {
       "File": "reparent.py"
     },
-    {
+    "vtdb": {
       "File": "vtdb_test.py"
     },
-    {
+    "vtgate_utils": {
       "File": "vtgate_utils_test.py"
     },
-    {
+    "rowcache_invalidator": {
       "File": "rowcache_invalidator.py"
     },
-    {
+    "vtgatev2": {
       "File": "vtgatev2_test.py"
     },
-    {
+    "zkocc": {
       "File": "zkocc_test.py"
     },
-    {
+    "initial_sharding_bytes": {
       "File": "initial_sharding_bytes.py"
     },
-    {
+    "initial_sharding": {
       "File": "initial_sharding.py"
     },
-    {
+    "resharding_bytes": {
       "File": "resharding_bytes.py"
     },
-    {
+    "resharding": {
       "File": "resharding.py"
     },
-    {
+    "worker": {
       "File": "worker.py"
     }
-  ]
+  }
 }