Add test for vstream with keyspaces_to_watch set

Signed-off-by: Matt Lord <mattalord@gmail.com>

.github/workflows/cluster_endtoend_vstream_with_keyspaces_to_watch.yml

@@ -0,0 +1,53 @@
+# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows"
+
+name: Cluster (vstream_with_keyspaces_to_watch)
+on: [push, pull_request]
+concurrency:
+  group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (vstream_with_keyspaces_to_watch)')
+  cancel-in-progress: true
+
+jobs:
+  build:
+    name: Run endtoend tests on Cluster (vstream_with_keyspaces_to_watch)
+    runs-on: ubuntu-18.04
+
+    steps:
+    - name: Set up Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.17
+
+    - name: Tune the OS
+      run: |
+        echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range
+
+    # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185
+    - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file
+      run: |
+        echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
+    # DON'T FORGET TO REMOVE CODE ABOVE WHEN ISSUE IS ADRESSED!
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata
+        sudo service mysql stop
+        sudo service etcd stop
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        go mod download
+
+        wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get install -y gnupg2
+        sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get update
+        sudo apt-get install percona-xtrabackup-24
+
+    - name: Run cluster endtoend test
+      timeout-minutes: 30
+      run: |
+        source build.env
+        eatmydata -- go run test.go -docker=false -print-log -follow -shard vstream_with_keyspaces_to_watch

go/test/endtoend/vreplication/cluster.go

@@ -26,6 +26,7 @@ var (
 	vtdataroot            string
 	mainClusterConfig     *ClusterConfig
 	externalClusterConfig *ClusterConfig
+	extraVTGateArgs       = []string{"-tablet_refresh_interval", "10ms"}
 )
 
 // ClusterConfig defines the parameters like ports, tmpDir, tablet types which uniquely define a vitess cluster
@@ -383,7 +384,7 @@ func (vc *VitessCluster) StartVtgate(t testing.TB, cell *Cell, cellsToWatch stri
 		vc.ClusterConfig.tabletTypes,
 		vc.ClusterConfig.topoPort,
 		vc.ClusterConfig.tmpDir,
-		[]string{"-tablet_refresh_interval", "10ms"},
+		extraVTGateArgs,
 		vc.ClusterConfig.vtgatePlannerVersion)
 	require.NotNil(t, vtgate)
 	if err := vtgate.Setup(); err != nil {

go/test/endtoend/vreplication/vstream_test.go

@@ -38,13 +38,15 @@ import (
 	"vitess.io/vitess/go/vt/vtgate/vtgateconn"
 )
 
-// Validates that a reparent while VStream API is streaming doesn't miss any events
-// We stream only from the primary and while streaming we reparent to a replica and then back to the original primary
-func TestVStreamFailover(t *testing.T) {
+// Validates that we have a working VStream API
+// If Failover is enabled:
+//   - We ensure that this works through active reparents and doesn't miss any events
+//   - We stream only from the primary and while streaming we reparent to a replica and then back to the original primary
+func testVStreamWithFailover(t *testing.T, failover bool) {
 	defaultCellName := "zone1"
 	cells := []string{"zone1"}
 	allCellNames = "zone1"
-	vc = NewVitessCluster(t, "TestVStreamFailover", cells, mainClusterConfig)
+	vc = NewVitessCluster(t, "TestVStreamWithFailover", cells, mainClusterConfig)
 
 	require.NotNil(t, vc)
 	defaultReplicas = 2
@@ -129,37 +131,40 @@ func TestVStreamFailover(t *testing.T) {
 		}
 	}()
 
-	// run two PRS after one second each, wait for events to be received and exit test
-	ticker := time.NewTicker(1 * time.Second)
-	tickCount := 0
-	// this for loop implements a mini state machine that does the two PRSs, waits a bit after the second PRS,
-	// stops the insertions, waits for a bit again for the vstream to catchup and signals the test to stop
-	for {
-		<-ticker.C
-		tickCount++
-		switch tickCount {
-		case 1:
-			insertMu.Lock()
-			output, err := vc.VtctlClient.ExecuteCommandWithOutput("PlannedReparentShard", "-keyspace_shard=product/0", "-new_primary=zone1-101")
-			insertMu.Unlock()
-			log.Infof("output of first PRS is %s", output)
-			require.NoError(t, err)
-		case 2:
-			insertMu.Lock()
-			output, err := vc.VtctlClient.ExecuteCommandWithOutput("PlannedReparentShard", "-keyspace_shard=product/0", "-new_primary=zone1-100")
-			insertMu.Unlock()
-			log.Infof("output of second PRS is %s", output)
-			require.NoError(t, err)
-			time.Sleep(100 * time.Millisecond)
-			stopInserting = true
-			time.Sleep(2 * time.Second)
-			done = true
-		}
+	if failover {
+		// run two PRS after one second each, wait for events to be received and exit test
+		ticker := time.NewTicker(1 * time.Second)
+		tickCount := 0
+		// this for loop implements a mini state machine that does the two PRSs, waits a bit after the second PRS,
+		// stops the insertions, waits for a bit again for the vstream to catchup and signals the test to stop
+		for {
+			<-ticker.C
+			tickCount++
+			switch tickCount {
+			case 1:
+				insertMu.Lock()
+				output, err := vc.VtctlClient.ExecuteCommandWithOutput("PlannedReparentShard", "-keyspace_shard=product/0", "-new_primary=zone1-101")
+				insertMu.Unlock()
+				log.Infof("output of first PRS is %s", output)
+				require.NoError(t, err)
+			case 2:
+				insertMu.Lock()
+				output, err := vc.VtctlClient.ExecuteCommandWithOutput("PlannedReparentShard", "-keyspace_shard=product/0", "-new_primary=zone1-100")
+				insertMu.Unlock()
+				log.Infof("output of second PRS is %s", output)
+				require.NoError(t, err)
+				time.Sleep(100 * time.Millisecond)
+				stopInserting = true
+				time.Sleep(2 * time.Second)
+				done = true
+			}
 
-		if done {
-			break
+			if done {
+				break
+			}
 		}
 	}
+
 	qr := execVtgateQuery(t, vtgateConn, "product", "select count(*) from customer")
 	require.NotNil(t, qr)
 	// total number of row events found by the VStream API should match the rows inserted
@@ -368,6 +373,10 @@ func testVStreamStopOnReshardFlag(t *testing.T, stopOnReshard bool, baseTabletID
 	return &ne
 }
 
+func TestVStreamFailover(t *testing.T) {
+	testVStreamWithFailover(t, true)
+}
+
 func TestVStreamStopOnReshardTrue(t *testing.T) {
 	ne := testVStreamStopOnReshardFlag(t, true, 1000)
 	require.Greater(t, ne.numJournalEvents, int64(0))
@@ -387,3 +396,11 @@ func TestVStreamStopOnReshardFalse(t *testing.T) {
 	require.NotZero(t, ne.numLessThan40Events)
 	require.NotZero(t, ne.numGreaterThan40Events)
 }
+
+func TestVStreamWithKeyspacesToWatch(t *testing.T) {
+	extraVTGateArgs = append(extraVTGateArgs, []string{
+		"-keyspaces_to_watch", "product",
+	}...)
+
+	testVStreamWithFailover(t, false)
+}

test/ci_workflow_gen.go

@@ -62,6 +62,7 @@ var (
 		"vstream_failover",
 		"vstream_stoponreshard_true",
 		"vstream_stoponreshard_false",
+		"vstream_with_keyspaces_to_watch",
 		"onlineddl_ghost",
 		"onlineddl_vrepl",
 		"onlineddl_vrepl_stress",

test/config.json

@@ -1013,6 +1013,15 @@
 			"RetryMax": 1,
 			"Tags": []
 		},
+		"vstream_with_keyspaces_to_watch": {
+			"File": "unused.go",
+			"Args": ["vitess.io/vitess/go/test/endtoend/vreplication", "-run", "VStreamWithKeyspacesToWatch"],
+			"Command": [],
+			"Manual": false,
+			"Shard": "vstream_with_keyspaces_to_watch",
+			"RetryMax": 1,
+			"Tags": []
+		},
 		"vtorc": {
 			"File": "unused.go",
 			"Args": ["vitess.io/vitess/go/test/endtoend/vtorc"],