Use VM and MySQL 8.0 template for cross db version test

Signed-off-by: Matt Lord <mattalord@gmail.com>

.github/workflows/cluster_endtoend_mysql80.yml

@@ -47,7 +47,7 @@ jobs:
         sudo apt-get update
 
         # Install everything else we need, and configure
-        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata
+        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils
         sudo service mysql stop
         sudo service etcd stop
         sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/

.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml

@@ -2,11 +2,19 @@
 
 name: Cluster (vreplication_across_db_versions)
 on: [push, pull_request]
+concurrency:
+  group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (vreplication_across_db_versions)')
+  cancel-in-progress: true
+
+env:
+  LAUNCHABLE_ORGANIZATION: "vitess"
+  LAUNCHABLE_WORKSPACE: "vitess-app"
+  GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}"
 
 jobs:
   build:
     name: Run endtoend tests on Cluster (vreplication_across_db_versions)
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
 
     steps:
     - name: Set up Go
@@ -14,6 +22,9 @@ jobs:
       with:
         go-version: 1.17
 
+    - name: Set up python
+      uses: actions/setup-python@v2
+
     - name: Tune the OS
       run: |
         echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range
@@ -23,10 +34,58 @@ jobs:
 
     - name: Get dependencies
       run: |
+        # Get key to latest MySQL repo
+        sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29
+
+        # Setup MySQL 8.0
+        wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb
+        echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections
+        sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config*
         sudo apt-get update
-        sudo apt-get install -y xz-utils
+
+        # Install everything else we need, and configure
+        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils
+        sudo service mysql stop
+        sudo service etcd stop
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        go mod download
+
+        # install JUnit report formatter
+        go get -u github.com/vitessio/go-junit-report@HEAD
+
+        wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get install -y gnupg2
+        sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get update
+        sudo apt-get install percona-xtrabackup-24
+
+    - name: Setup launchable dependencies
+      run: |
+        # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up
+        pip3 install --user launchable~=1.0 > /dev/null
+
+        # verify that launchable setup is all correct.
+        launchable verify || true
+
+        # Tell Launchable about the build you are producing and testing
+        launchable record build --name "$GITHUB_RUN_ID" --source .
 
     - name: Run cluster endtoend test
       timeout-minutes: 30
       run: |
-        go run test.go -docker=true --follow -shard vreplication_across_db_versions
+        source build.env
+
+        set -x
+
+        # run the tests however you normally do, then produce a JUnit XML file
+        eatmydata -- go run test.go -docker=false -follow -shard vreplication_across_db_versions | tee -a output.txt | go-junit-report -set-exit-code > report.xml
+
+    - name: Print test output and Record test result in launchable
+      run: |
+        # send recorded tests to launchable
+        launchable record tests --build "$GITHUB_RUN_ID" go-test . || true
+
+        # print test output
+        cat output.txt
+      if: always()

go/test/endtoend/vreplication/cluster.go

@@ -170,13 +170,13 @@ func getDBMajorVersionInstalled() string {
 
 func downloadDBVersion(dbType string, majorVersion string, path string) error {
 	client := http.Client{
-		Timeout: 60 * time.Second,
+		Timeout: 10 * time.Minute,
 	}
 	var url, file, versionFile string
 	dbType = strings.ToLower(dbType)
 
 	if dbType == "mysql" && majorVersion == "5.7" {
-		versionFile = "mysql-5.7.37-linux-glibc2.12-x86_64.tar"
+		versionFile = "mysql-5.7.37-linux-glibc2.12-x86_64.tar.gz"
 		url = "https://dev.mysql.com/get/Downloads/MySQL-5.7/" + versionFile
 	} else if dbType == "mysql" && majorVersion == "8.0" {
 		versionFile = "mysql-8.0.28-linux-glibc2.17-x86_64-minimal.tar.xz"
@@ -191,15 +191,18 @@ func downloadDBVersion(dbType string, majorVersion string, path string) error {
 	}
 	resp, err := client.Get(url)
 	if err != nil {
-		return err
+		return fmt.Errorf("error downloading contents of %s to %s. Error: %v", url, file, err)
 	}
 	defer resp.Body.Close()
 	out, err := os.Create(file)
 	if err != nil {
-		return err
+		return fmt.Errorf("error creating file %s to save the contents of %s. Error: %v", file, url, err)
 	}
 	defer out.Close()
-	io.Copy(out, resp.Body)
+	_, err = io.Copy(out, resp.Body)
+	if err != nil {
+		return fmt.Errorf("error saving contents of %s to %s. Error: %v", url, file, err)
+	}
 
 	untarCmd := exec.Command("/bin/sh", "-c", fmt.Sprintf("tar xvf %s -C %s --strip-components=1", file, path))
 	output, err := untarCmd.CombinedOutput()
@@ -314,7 +317,7 @@ func (vc *VitessCluster) AddKeyspace(t *testing.T, cells []*Cell, ksName string,
 					t.Logf("Requsted database version %s is already installed, doing nothing.", dbTypeMajorVersion)
 					continue
 				}
-				path := fmt.Sprintf("/vt/%s", dbTypeMajorVersion)
+				path := fmt.Sprintf("/tmp/%s", dbTypeMajorVersion)
 				// Set the root path and create it if needed
 				if err := setVtMySQLRoot(path); err != nil {
 					t.Fatalf("Could not set VT_MYSQL_ROOT to %s, error: %v", path, err)

test/ci_workflow_gen.go

@@ -101,6 +101,7 @@ var (
 		"resharding",
 		"resharding_bytes",
 		"mysql80",
+		"vreplication_across_db_versions",
 		"vreplication_multicell",
 		"vreplication_cellalias",
 		"vtorc",
@@ -110,7 +111,6 @@ var (
 	clusterDockerList     = []string{
 		"vreplication_basic",
 		"vreplication_v2",
-		"vreplication_across_db_versions",
 	}
 	// TODO: currently some percona tools including xtrabackup are installed on all clusters, we can possibly optimize
 	// this by only installing them in the required clusters
@@ -123,6 +123,7 @@ var (
 	}
 	clustersRequiringMySQL80 = []string{
 		"mysql80",
+		"vreplication_across_db_versions",
 	}
 )
 

test/templates/cluster_endtoend_test_mysql80.tpl

@@ -45,7 +45,7 @@ jobs:
         sudo apt-get update
 
         # Install everything else we need, and configure
-        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata
+        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils
         sudo service mysql stop
         sudo service etcd stop
         sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/