diff --git a/.github/workflows/cluster_endtoend_mysql80.yml b/.github/workflows/cluster_endtoend_mysql80.yml index 4cfd807d0c..1343b212ca 100644 --- a/.github/workflows/cluster_endtoend_mysql80.yml +++ b/.github/workflows/cluster_endtoend_mysql80.yml @@ -47,7 +47,7 @@ jobs: sudo apt-get update # Install everything else we need, and configure - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml b/.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml index 8ddc9404f3..c60b2e18b5 100644 --- a/.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml +++ b/.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml @@ -2,11 +2,19 @@ name: Cluster (vreplication_across_db_versions) on: [push, pull_request] +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (vreplication_across_db_versions)') + cancel-in-progress: true + +env: + LAUNCHABLE_ORGANIZATION: "vitess" + LAUNCHABLE_WORKSPACE: "vitess-app" + GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}" jobs: build: name: Run endtoend tests on Cluster (vreplication_across_db_versions) - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - name: Set up Go @@ -14,6 +22,9 @@ jobs: with: go-version: 1.17 + - name: Set up python + uses: actions/setup-python@v2 + - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range @@ -23,10 +34,58 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y xz-utils + + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + # install JUnit report formatter + go get -u github.com/vitessio/go-junit-report@HEAD + + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get install -y gnupg2 + sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get update + sudo apt-get install percona-xtrabackup-24 + + - name: Setup launchable dependencies + run: | + # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up + pip3 install --user launchable~=1.0 > /dev/null + + # verify that launchable setup is all correct. + launchable verify || true + + # Tell Launchable about the build you are producing and testing + launchable record build --name "$GITHUB_RUN_ID" --source . - name: Run cluster endtoend test timeout-minutes: 30 run: | - go run test.go -docker=true --follow -shard vreplication_across_db_versions + source build.env + + set -x + + # run the tests however you normally do, then produce a JUnit XML file + eatmydata -- go run test.go -docker=false -follow -shard vreplication_across_db_versions | tee -a output.txt | go-junit-report -set-exit-code > report.xml + + - name: Print test output and Record test result in launchable + run: | + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + + # print test output + cat output.txt + if: always() diff --git a/go/test/endtoend/vreplication/cluster.go b/go/test/endtoend/vreplication/cluster.go index 00cf1c30d5..55c8b28cdc 100644 --- a/go/test/endtoend/vreplication/cluster.go +++ b/go/test/endtoend/vreplication/cluster.go @@ -170,13 +170,13 @@ func getDBMajorVersionInstalled() string { func downloadDBVersion(dbType string, majorVersion string, path string) error { client := http.Client{ - Timeout: 60 * time.Second, + Timeout: 10 * time.Minute, } var url, file, versionFile string dbType = strings.ToLower(dbType) if dbType == "mysql" && majorVersion == "5.7" { - versionFile = "mysql-5.7.37-linux-glibc2.12-x86_64.tar" + versionFile = "mysql-5.7.37-linux-glibc2.12-x86_64.tar.gz" url = "https://dev.mysql.com/get/Downloads/MySQL-5.7/" + versionFile } else if dbType == "mysql" && majorVersion == "8.0" { versionFile = "mysql-8.0.28-linux-glibc2.17-x86_64-minimal.tar.xz" @@ -191,15 +191,18 @@ func downloadDBVersion(dbType string, majorVersion string, path string) error { } resp, err := client.Get(url) if err != nil { - return err + return fmt.Errorf("error downloading contents of %s to %s. Error: %v", url, file, err) } defer resp.Body.Close() out, err := os.Create(file) if err != nil { - return err + return fmt.Errorf("error creating file %s to save the contents of %s. Error: %v", file, url, err) } defer out.Close() - io.Copy(out, resp.Body) + _, err = io.Copy(out, resp.Body) + if err != nil { + return fmt.Errorf("error saving contents of %s to %s. Error: %v", url, file, err) + } untarCmd := exec.Command("/bin/sh", "-c", fmt.Sprintf("tar xvf %s -C %s --strip-components=1", file, path)) output, err := untarCmd.CombinedOutput() @@ -314,7 +317,7 @@ func (vc *VitessCluster) AddKeyspace(t *testing.T, cells []*Cell, ksName string, t.Logf("Requsted database version %s is already installed, doing nothing.", dbTypeMajorVersion) continue } - path := fmt.Sprintf("/vt/%s", dbTypeMajorVersion) + path := fmt.Sprintf("/tmp/%s", dbTypeMajorVersion) // Set the root path and create it if needed if err := setVtMySQLRoot(path); err != nil { t.Fatalf("Could not set VT_MYSQL_ROOT to %s, error: %v", path, err) diff --git a/test/ci_workflow_gen.go b/test/ci_workflow_gen.go index b06622f877..d8f66cb352 100644 --- a/test/ci_workflow_gen.go +++ b/test/ci_workflow_gen.go @@ -101,6 +101,7 @@ var ( "resharding", "resharding_bytes", "mysql80", + "vreplication_across_db_versions", "vreplication_multicell", "vreplication_cellalias", "vtorc", @@ -110,7 +111,6 @@ var ( clusterDockerList = []string{ "vreplication_basic", "vreplication_v2", - "vreplication_across_db_versions", } // TODO: currently some percona tools including xtrabackup are installed on all clusters, we can possibly optimize // this by only installing them in the required clusters @@ -123,6 +123,7 @@ var ( } clustersRequiringMySQL80 = []string{ "mysql80", + "vreplication_across_db_versions", } ) diff --git a/test/templates/cluster_endtoend_test_mysql80.tpl b/test/templates/cluster_endtoend_test_mysql80.tpl index b91a8743fe..806d079be5 100644 --- a/test/templates/cluster_endtoend_test_mysql80.tpl +++ b/test/templates/cluster_endtoend_test_mysql80.tpl @@ -45,7 +45,7 @@ jobs: sudo apt-get update # Install everything else we need, and configure - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/