Enable VTOrc in backup tests (#11410)

* feat: enable vtorc in backup tests

Signed-off-by: Manan Gupta <manan@planetscale.com>

* feat: reorder verifying initial replication to happen first

Signed-off-by: Manan Gupta <manan@planetscale.com>

* test: increase timeout for VTOrc recovery to be successful and improve VTOrc performance

Signed-off-by: Manan Gupta <manan@planetscale.com>

Signed-off-by: Manan Gupta <manan@planetscale.com>

go/test/endtoend/backup/vtctlbackup/backup_utils.go

@@ -763,10 +763,21 @@ func terminatedRestore(t *testing.T) {
 // Args:
 // tablet_type: 'replica' or 'rdonly'.
 func vtctlBackup(t *testing.T, tabletType string) {
-	restoreWaitForBackup(t, tabletType, nil, true)
-	verifyInitialReplication(t)
+	// Start vtorc before running backups
+	vtorcProcess := localCluster.NewVTOrcProcess(cluster.VTOrcConfiguration{})
+	err := vtorcProcess.Setup()
+	require.NoError(t, err)
+	localCluster.VTOrcProcesses = append(localCluster.VTOrcProcesses, vtorcProcess)
 
-	err := localCluster.VtctlclientProcess.ExecuteCommand("Backup", replica1.Alias)
+	// StopReplication on replica1. We verify that the replication works fine later in
+	// verifyInitialReplication. So this will also check that VTOrc is running.
+	err = localCluster.VtctlclientProcess.ExecuteCommand("StopReplication", replica1.Alias)
+	require.Nil(t, err)
+
+	verifyInitialReplication(t)
+	restoreWaitForBackup(t, tabletType, nil, true)
+
+	err = localCluster.VtctlclientProcess.ExecuteCommand("Backup", replica1.Alias)
 	require.Nil(t, err)
 
 	backups := localCluster.VerifyBackupCount(t, shardKsName, 1)
@@ -781,6 +792,11 @@ func vtctlBackup(t *testing.T, tabletType string) {
 	cluster.VerifyLocalMetadata(t, replica2, keyspaceName, shardName, cell)
 	verifyAfterRemovingBackupNoBackupShouldBePresent(t, backups)
 
+	// Stop VTOrc
+	err = localCluster.VTOrcProcesses[0].TearDown()
+	localCluster.VTOrcProcesses = nil
+	require.NoError(t, err)
+
 	err = replica2.VttabletProcess.TearDown()
 	require.Nil(t, err)
 

go/test/endtoend/cluster/cluster_util.go

@@ -94,7 +94,7 @@ func GetPrimaryPosition(t *testing.T, vttablet Vttablet, hostname string) (strin
 // VerifyRowsInTabletForTable verifies the total number of rows in a table.
 // This is used to check that replication has caught up with the changes on primary.
 func VerifyRowsInTabletForTable(t *testing.T, vttablet *Vttablet, ksName string, expectedRows int, tableName string) {
-	timeout := time.Now().Add(10 * time.Second)
+	timeout := time.Now().Add(1 * time.Minute)
 	for time.Now().Before(timeout) {
 		// ignoring the error check, if the newly created table is not replicated, then there might be error and we should ignore it
 		// but eventually it will catch up and if not caught up in required time, testcase will fail

go/test/endtoend/cluster/vtorc_process.go

@@ -112,6 +112,8 @@ func (orc *VTOrcProcess) Setup() (err error) {
 		// This parameter is overriden from the config file, added here to just verify that we indeed use the config file paramter over the flag
 		"--recovery-period-block-duration", "10h",
 		"--instance-poll-time", "1s",
+		// Faster topo information refresh speeds up the tests. This doesn't add any significant load either
+		"--topo-information-refresh-duration", "3s",
 		"--orc_web_dir", path.Join(os.Getenv("VTROOT"), "web", "vtorc"),
 	)
 	if *isCoverage {