Merge pull request #5559 from planetscale/morgo-matrix-unit

Add matrix build for unit tests

.github/workflows/check_make_parser.yml

@@ -13,11 +13,11 @@ jobs:
         go-version: 1.13
 
     - name: Check out code
-      uses: actions/checkout@v1
+      uses: actions/checkout@v2
 
     - name: Get dependencies
       run: |
-        sudo apt-get update || echo "update failed" 
+        sudo apt-get update
         sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget
         sudo service mysql stop
         sudo service etcd stop

.github/workflows/cluster_endtoend.yml

@@ -13,11 +13,11 @@ jobs:
         go-version: 1.13
 
     - name: Check out code
-      uses: actions/checkout@v1
+      uses: actions/checkout@v2
 
     - name: Get dependencies
       run: |
-        sudo apt-get update || echo "update failed"
+        sudo apt-get update
         sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget
         sudo service mysql stop
         sudo service etcd stop

.github/workflows/e2e_race.yml

@@ -13,11 +13,11 @@ jobs:
         go-version: 1.13
 
     - name: Check out code
-      uses: actions/checkout@v1
+      uses: actions/checkout@v2
 
     - name: Get dependencies
       run: |
-        sudo apt-get update || echo "update failed"
+        sudo apt-get update
         sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget
         sudo service mysql stop
         sudo service etcd stop

.github/workflows/endtoend.yml

@@ -13,11 +13,11 @@ jobs:
         go-version: 1.13
 
     - name: Check out code
-      uses: actions/checkout@v1
+      uses: actions/checkout@v2
 
     - name: Get dependencies
       run: |
-        sudo apt-get update || echo "update failed"
+        sudo apt-get update
         sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget
         sudo service mysql stop
         sudo service etcd stop

.github/workflows/local_example.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-latest]
+        os: [ubuntu-latest]
 
     steps:
 
@@ -17,12 +17,12 @@ jobs:
         go-version: 1.13
 
     - name: Check out code
-      uses: actions/checkout@v1
+      uses: actions/checkout@v2
 
     - name: Get dependencies
       run: |
         if [ ${{matrix.os}} = "ubuntu-latest" ]; then
-          sudo apt-get update || echo "update failed"
+          sudo apt-get update
           sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget
           sudo service mysql stop
           sudo service etcd stop

.github/workflows/unit.yml

@@ -3,28 +3,75 @@ on: [push, pull_request]
 jobs:
 
   build:
-    name: Unit Test
     runs-on: ubuntu-latest
-    steps:
+    strategy:
+      matrix:
+        name: [mysql57, mysql80]
 
+    steps:
     - name: Set up Go
       uses: actions/setup-go@v1
       with:
         go-version: 1.12
 
     - name: Check out code
-      uses: actions/checkout@v1
+      uses: actions/checkout@v2
 
     - name: Get dependencies
       run: |
-        sudo apt-get update || echo "update failed"
-        sudo apt-get install -y mysql-server mysql-client make unzip g++ curl git wget ant openjdk-8-jdk
+        sudo apt-get update
+
+        if [ ${{matrix.name}} = "mysql57" ]; then
+          sudo apt-get install -y mysql-server mysql-client
+        else
+          # Uninstall likely installed MySQL first
+          sudo apt-get remove -y mysql-server mysql-client
+          
+          if [ ${{matrix.name}} = "percona56" ]; then
+            # Currently this fails on vt/vttablet/tabletserver/vstreamer
+            # Once we fix issue #5571 we can enable it.
+            sudo rm -rf /var/lib/mysql
+            wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
+            sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
+            sudo apt update
+            sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y percona-server-server-5.6 percona-server-client-5.6  
+          elif [ ${{matrix.name}} = "mysql80" ]; then
+            wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb
+            echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections 
+            sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config*
+            sudo apt-get update
+            sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client
+          elif [ ${{matrix.name}} = "mariadb101" ]; then
+            sudo apt install -y mariadb-server mariadb-client
+          elif [ ${{matrix.name}} = "mariadb102" ]; then
+            # Currently this fails on vitess.io/vitess/go/mysql
+            # Once we fix issue #5569 we can enable it.
+            sudo apt-get install -y software-properties-common
+            sudo apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xF1656F24C74CD1D8
+            sudo add-apt-repository 'deb [arch=amd64,arm64,ppc64el] http://nyc2.mirrors.digitalocean.com/mariadb/repo/10.2/ubuntu bionic main'
+            sudo apt update
+            sudo DEBIAN_FRONTEND="noninteractive" apt install -y mariadb-server
+          elif [ ${{matrix.name}} = "mariadb103" ]; then
+            # Currently this fails on vitess.io/vitess/go/mysql
+            # Once we fix issue #5569 we can enable it.
+            sudo apt-get install -y software-properties-common
+            sudo apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xF1656F24C74CD1D8
+            sudo add-apt-repository 'deb [arch=amd64,arm64,ppc64el] http://nyc2.mirrors.digitalocean.com/mariadb/repo/10.3/ubuntu bionic main'
+            sudo apt update
+            sudo DEBIAN_FRONTEND="noninteractive" apt install -y mariadb-server
+          fi
+        fi
+
+        sudo apt-get install -y make unzip g++ curl git wget ant openjdk-8-jdk
         sudo service mysql stop
+        sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263
         sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
-        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile"
+
         mkdir -p dist bin
         curl -L https://github.com/coreos/etcd/releases/download/v3.3.10/etcd-v3.3.10-linux-amd64.tar.gz | tar -zxC dist
         mv dist/etcd-v3.3.10-linux-amd64/{etcd,etcdctl} bin/
+
         go mod download
 
     - name: Run make tools

.github/workflows/unit_race.yml

@@ -1,5 +1,5 @@
 name: unit_race
-on: [push]
+on: [repository_dispatch] # disable for now
 jobs:
 
   build:
@@ -13,16 +13,20 @@ jobs:
         go-version: 1.12
 
     - name: Check out code
-      uses: actions/checkout@v1
+      uses: actions/checkout@v2
 
     - name: Get dependencies
       run: |
-        sudo apt-get update || echo "update failed"
-        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget
+        sudo apt-get update
+        sudo apt-get install -y mysql-server mysql-client make unzip g++ curl git wget
         sudo service mysql stop
-        sudo service etcd stop
         sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
         sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+
+        mkdir -p dist bin
+        curl -L https://github.com/coreos/etcd/releases/download/v3.3.10/etcd-v3.3.10-linux-amd64.tar.gz | tar -zxC dist
+        mv dist/etcd-v3.3.10-linux-amd64/{etcd,etcdctl} bin/
+
         go mod download
 
     - name: Run make tools

examples/local/env.sh

@@ -15,9 +15,6 @@
 # limitations under the License.
 
 hostname=`hostname -f`
-if [ $(uname) == "Darwin" ]; then
-  hostname="localhost"
-fi
 vtctld_web_port=15000
 export VTDATAROOT="${VTDATAROOT:-${VTROOT}/vtdataroot}"
 

tools/unit_test_race.sh

@@ -27,15 +27,39 @@ fi
 
 packages_with_tests=$(go list -f '{{if len .TestGoFiles}}{{.ImportPath}} {{join .TestGoFiles " "}}{{end}}' ./go/vt/... | sort)
 
-# endtoend tests should be in a directory called endtoend
-all_except_e2e_tests=$(echo "$packages_with_tests" | cut -d" " -f1 | grep -v "endtoend")
+# exclude end to end tests
+packages_to_test=$(echo "$packages_with_tests" | cut -d" " -f1 | grep -v "endtoend")
+all_except_flaky_tests=$(echo "$packages_to_test" | grep -vE ".+ .+_flaky_test\.go" | cut -d" " -f1 | grep -v "endtoend")
+flaky_tests=$(echo "$packages_to_test" | grep -E ".+ .+_flaky_test\.go" | cut -d" " -f1)
 
-# Run non endtoend tests.
-echo "$all_except_e2e_tests" | xargs go test $VT_GO_PARALLEL -race
+# Flaky tests have the suffix "_flaky_test.go".
+# Exclude endtoend tests
+all_except_flaky_tests=$(echo "$packages_with_tests" | grep -vE ".+ .+_flaky_test\.go" | cut -d" " -f1 | grep -v "endtoend")
+flaky_tests=$(echo "$packages_with_tests" | grep -E ".+ .+_flaky_test\.go" | cut -d" " -f1)
 
+# Run non-flaky tests.
+echo "$all_except_flaky_tests" | xargs go test $VT_GO_PARALLEL -race -count=1
 if [ $? -ne 0 ]; then
-  echo "WARNING: POSSIBLE DATA RACE"
+  echo "ERROR: Go unit tests failed. See above for errors."
   echo
-  echo "ERROR: go test -race failed. See log above."
+  echo "This should NOT happen. Did you introduce a flaky unit test?"
+  echo "If so, please rename it to the suffix _flaky_test.go."
   exit 1
 fi
+
+echo '# Flaky tests (3 attempts permitted)'
+
+# Run flaky tests sequentially. Retry when necessary.
+for pkg in $flaky_tests; do
+  max_attempts=3
+  attempt=1
+  # Set a timeout because some tests may deadlock when they flake.
+  until go test -timeout 30s $VT_GO_PARALLEL $pkg -race -count=1; do
+    echo "FAILED (try $attempt/$max_attempts) in $pkg (return code $?). See above for errors."
+    if [ $((++attempt)) -gt $max_attempts ]; then
+      echo "ERROR: Flaky Go unit tests in package $pkg failed too often (after $max_attempts retries). Please reduce the flakiness."
+      exit 1
+    fi
+  done
+done
+

tools/unit_test_runner.sh

@@ -46,7 +46,7 @@ all_except_flaky_tests=$(echo "$packages_with_tests" | grep -vE ".+ .+_flaky_tes
 flaky_tests=$(echo "$packages_with_tests" | grep -E ".+ .+_flaky_test\.go" | cut -d" " -f1)
 
 # Run non-flaky tests.
-echo "$all_except_flaky_tests" | xargs go test $VT_GO_PARALLEL
+echo "$all_except_flaky_tests" | xargs go test $VT_GO_PARALLEL -count=1
 if [ $? -ne 0 ]; then
   echo "ERROR: Go unit tests failed. See above for errors."
   echo
@@ -62,7 +62,7 @@ for pkg in $flaky_tests; do
   max_attempts=3
   attempt=1
   # Set a timeout because some tests may deadlock when they flake.
-  until go test -timeout 30s $VT_GO_PARALLEL $pkg; do
+  until go test -timeout 30s $VT_GO_PARALLEL $pkg -count=1; do
     echo "FAILED (try $attempt/$max_attempts) in $pkg (return code $?). See above for errors."
     if [ $((++attempt)) -gt $max_attempts ]; then
       echo "ERROR: Flaky Go unit tests in package $pkg failed too often (after $max_attempts retries). Please reduce the flakiness."