From 22bec26f6d899ecb902c6b16124bd12172871967 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Fri, 28 May 2021 20:10:18 -0400 Subject: [PATCH 01/64] Move duplicated vtctld cert/tls flags to a common package Signed-off-by: Andrew Mason --- go/vt/vtctl/grpcclientcommon/dial_option.go | 41 +++++++++++++++++++++ go/vt/vtctl/grpcvtctlclient/client.go | 11 +----- go/vt/vtctl/grpcvtctldclient/client.go | 21 +---------- 3 files changed, 45 insertions(+), 28 deletions(-) create mode 100644 go/vt/vtctl/grpcclientcommon/dial_option.go diff --git a/go/vt/vtctl/grpcclientcommon/dial_option.go b/go/vt/vtctl/grpcclientcommon/dial_option.go new file mode 100644 index 0000000000..7a69bcf963 --- /dev/null +++ b/go/vt/vtctl/grpcclientcommon/dial_option.go @@ -0,0 +1,41 @@ +/* +Copyright 2021 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package grpcclientcommon defines the flags shared by both grpcvtctlclient and +// grpcvtctldclient. +package grpcclientcommon + +import ( + "flag" + + "google.golang.org/grpc" + + "vitess.io/vitess/go/vt/grpcclient" +) + +var ( + cert = flag.String("vtctld_grpc_cert", "", "the cert to use to connect") + key = flag.String("vtctld_grpc_key", "", "the key to use to connect") + ca = flag.String("vtctld_grpc_ca", "", "the server ca to use to validate servers when connecting") + name = flag.String("vtctld_grpc_server_name", "", "the server name to use to validate server certificate") +) + +// SecureDialOption returns a grpc.DialOption configured to use TLS (or +// insecure if no flags were set) based on the vtctld_grpc_* flags declared by +// this package. +func SecureDialOption() (grpc.DialOption, error) { + return grpcclient.SecureDialOption(*cert, *key, *ca, *name) +} diff --git a/go/vt/vtctl/grpcvtctlclient/client.go b/go/vt/vtctl/grpcvtctlclient/client.go index 5a61d7dc79..f0fe94ca33 100644 --- a/go/vt/vtctl/grpcvtctlclient/client.go +++ b/go/vt/vtctl/grpcvtctlclient/client.go @@ -18,7 +18,6 @@ limitations under the License. package grpcvtctlclient import ( - "flag" "time" "context" @@ -27,6 +26,7 @@ import ( "vitess.io/vitess/go/vt/grpcclient" "vitess.io/vitess/go/vt/logutil" + "vitess.io/vitess/go/vt/vtctl/grpcclientcommon" "vitess.io/vitess/go/vt/vtctl/vtctlclient" logutilpb "vitess.io/vitess/go/vt/proto/logutil" @@ -34,20 +34,13 @@ import ( vtctlservicepb "vitess.io/vitess/go/vt/proto/vtctlservice" ) -var ( - cert = flag.String("vtctld_grpc_cert", "", "the cert to use to connect") - key = flag.String("vtctld_grpc_key", "", "the key to use to connect") - ca = flag.String("vtctld_grpc_ca", "", "the server ca to use to validate servers when connecting") - name = flag.String("vtctld_grpc_server_name", "", "the server name to use to validate server certificate") -) - type gRPCVtctlClient struct { cc *grpc.ClientConn c vtctlservicepb.VtctlClient } func gRPCVtctlClientFactory(addr string) (vtctlclient.VtctlClient, error) { - opt, err := grpcclient.SecureDialOption(*cert, *key, *ca, *name) + opt, err := grpcclientcommon.SecureDialOption() if err != nil { return nil, err } diff --git a/go/vt/vtctl/grpcvtctldclient/client.go b/go/vt/vtctl/grpcvtctldclient/client.go index a0f5a14ef7..45e19ff88e 100644 --- a/go/vt/vtctl/grpcvtctldclient/client.go +++ b/go/vt/vtctl/grpcvtctldclient/client.go @@ -19,11 +19,10 @@ limitations under the License. package grpcvtctldclient import ( - "flag" - "google.golang.org/grpc" "vitess.io/vitess/go/vt/grpcclient" + "vitess.io/vitess/go/vt/vtctl/grpcclientcommon" "vitess.io/vitess/go/vt/vtctl/vtctldclient" vtctlservicepb "vitess.io/vitess/go/vt/proto/vtctlservice" @@ -31,22 +30,6 @@ import ( const connClosedMsg = "grpc: the client connection is closed" -// (TODO:@amason) - These flags match exactly the flags used in grpcvtctlclient. -// If a program attempts to import both of these packages, it will panic during -// startup due to the duplicate flags. -// -// For everything else I've been doing a sed s/vtctl/vtctld, but I cannot do -// that here, since these flags are already "vtctld_*". My other options are to -// name them "vtctld2_*" or to omit them completely. -// -// Not to pitch project ideas in comments, but a nice project to solve -var ( - cert = flag.String("vtctld_grpc_cert", "", "the cert to use to connect") - key = flag.String("vtctld_grpc_key", "", "the key to use to connect") - ca = flag.String("vtctld_grpc_ca", "", "the server ca to use to validate servers when connecting") - name = flag.String("vtctld_grpc_server_name", "", "the server name to use to validate server certificate") -) - type gRPCVtctldClient struct { cc *grpc.ClientConn c vtctlservicepb.VtctldClient @@ -56,7 +39,7 @@ type gRPCVtctldClient struct { //go:generate grpcvtctldclient -out client_gen.go func gRPCVtctldClientFactory(addr string) (vtctldclient.VtctldClient, error) { - opt, err := grpcclient.SecureDialOption(*cert, *key, *ca, *name) + opt, err := grpcclientcommon.SecureDialOption() if err != nil { return nil, err } From ea4926b2382e3df270bdba003295850f81362893 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Fri, 28 May 2021 20:47:41 -0400 Subject: [PATCH 02/64] Don't show `help requested` errors as errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This turns the output from: ``` ❯ vtctlclient -server "localhost:15999" ListAllTablets -h Usage: ListAllTablets , , ... Lists all tablets in an awk-friendly way. ListAllTablets Error: rpc error: code = Unknown desc = flag: help requested E0528 19:52:46.564223 77260 main.go:76] remote error: rpc error: code = Unknown desc = flag: help requested ❯ echo $? 1 ``` into: ``` ❯ vtctlclient -server "localhost:15999" ListAllTablets -h Usage: ListAllTablets , , ... Lists all tablets in an awk-friendly way. ❯ echo $? 0 ``` Signed-off-by: Andrew Mason --- go/cmd/vtctlclient/main.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/go/cmd/vtctlclient/main.go b/go/cmd/vtctlclient/main.go index e7311a5cff..bd8871851f 100644 --- a/go/cmd/vtctlclient/main.go +++ b/go/cmd/vtctlclient/main.go @@ -67,6 +67,10 @@ func main() { logutil.LogEvent(logger, e) }) if err != nil { + if strings.Contains(err.Error(), "flag: help requested") { + return + } + errStr := strings.Replace(err.Error(), "remote error: ", "", -1) fmt.Printf("%s Error: %s\n", flag.Arg(0), errStr) log.Error(err) From 1928fac2f79b7d6e115162dc2d480d0851b70b3f Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Fri, 28 May 2021 21:37:39 -0400 Subject: [PATCH 03/64] Add LegacyVtctlCommand shim to vtctldclient CLI This is admittedly pretty hacky, and I had to spend a lot of time messing around with cobra to make this behavior correctly. But, it works! Signed-off-by: Andrew Mason --- .../internal/command/legacy_shim.go | 101 ++++++++++++++++++ go/cmd/vtctldclient/internal/command/root.go | 16 ++- go/cmd/vtctldclient/plugin_grpcvtctlclient.go | 23 ++++ 3 files changed, 136 insertions(+), 4 deletions(-) create mode 100644 go/cmd/vtctldclient/internal/command/legacy_shim.go create mode 100644 go/cmd/vtctldclient/plugin_grpcvtctlclient.go diff --git a/go/cmd/vtctldclient/internal/command/legacy_shim.go b/go/cmd/vtctldclient/internal/command/legacy_shim.go new file mode 100644 index 0000000000..0d6bd8494b --- /dev/null +++ b/go/cmd/vtctldclient/internal/command/legacy_shim.go @@ -0,0 +1,101 @@ +/* +Copyright 2021 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package command + +import ( + "context" + "flag" + "fmt" + "strings" + + "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/util/sets" + + "vitess.io/vitess/go/cmd/vtctldclient/cli" + "vitess.io/vitess/go/vt/log" + "vitess.io/vitess/go/vt/logutil" + "vitess.io/vitess/go/vt/vtctl/vtctlclient" + + logutilpb "vitess.io/vitess/go/vt/proto/logutil" +) + +var ( + // LegacyVtctlCommand provides a shim to make legacy ExecuteVtctlCommand + // RPCs. This allows users to use a single binary to make RPCs against both + // the new and old vtctld gRPC APIs. + LegacyVtctlCommand = &cobra.Command{ + Use: "LegacyVtctlCommand", + Short: "Invoke a legacy vtctlclient command. Flag parsing is best effort.", + Args: cobra.ArbitraryArgs, + RunE: func(cmd *cobra.Command, args []string) error { + cli.FinishedParsing(cmd) + return runLegacyCommand(args) + }, + } +) + +func runLegacyCommand(args []string) error { + // Duplicated (mostly) from go/cmd/vtctlclient/main.go. + logger := logutil.NewConsoleLogger() + + ctx, cancel := context.WithTimeout(context.Background(), actionTimeout) + defer cancel() + + err := vtctlclient.RunCommandAndWait(ctx, server, args, func(e *logutilpb.Event) { + logutil.LogEvent(logger, e) + }) + if err != nil { + if strings.Contains(err.Error(), "flag: help requested") { + // Help is caught by SetHelpFunc, so we don't want to indicate this as an error. + return nil + } + + errStr := strings.Replace(err.Error(), "remote error: ", "", -1) + fmt.Printf("%s Error: %s\n", flag.Arg(0), errStr) + log.Error(err) + } + + return err +} + +func init() { + LegacyVtctlCommand.SetHelpFunc(func(cmd *cobra.Command, args []string) { + // PreRun (and PersistentPreRun) do not run when a Help command is + // being executed, so we need to duplicate the `--server` flag check + // here before we attempt to invoke the legacy help command. + if err := ensureServerArg(); err != nil { + log.Error(err) + return + } + + realArgs := cmd.Flags().Args() + + if len(realArgs) == 0 { + realArgs = append(realArgs, "help") + } + + argSet := sets.NewString(realArgs...) + if !argSet.HasAny("help", "-h", "--help") { + // Cobra tends to swallow the help flag, so we need to put it back + // into the arg slice that we pass to runLegacyCommand. + realArgs = append(realArgs, "-h") + } + + _ = runLegacyCommand(realArgs) + }) + Root.AddCommand(LegacyVtctlCommand) +} diff --git a/go/cmd/vtctldclient/internal/command/root.go b/go/cmd/vtctldclient/internal/command/root.go index 7243c8836e..8335710f44 100644 --- a/go/cmd/vtctldclient/internal/command/root.go +++ b/go/cmd/vtctldclient/internal/command/root.go @@ -25,7 +25,6 @@ import ( "github.com/spf13/cobra" "vitess.io/vitess/go/trace" - "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/vtctl/vtctldclient" ) @@ -44,9 +43,7 @@ var ( // command context for every command. PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) { traceCloser = trace.StartTracing("vtctldclient") - if server == "" { - err = errors.New("please specify -server to specify the vtctld server to connect to") - log.Error(err) + if err := ensureServerArg(); err != nil { return err } @@ -75,6 +72,17 @@ var ( } ) +var errNoServer = errors.New("please specify -server to specify the vtctld server to connect to") + +// ensureServerArg validates that --server was passed to the CLI. +func ensureServerArg() error { + if server == "" { + return errNoServer + } + + return nil +} + func init() { Root.PersistentFlags().StringVar(&server, "server", "", "server to use for connection") Root.PersistentFlags().DurationVar(&actionTimeout, "action_timeout", time.Hour, "timeout for the total command") diff --git a/go/cmd/vtctldclient/plugin_grpcvtctlclient.go b/go/cmd/vtctldclient/plugin_grpcvtctlclient.go new file mode 100644 index 0000000000..e08d657f42 --- /dev/null +++ b/go/cmd/vtctldclient/plugin_grpcvtctlclient.go @@ -0,0 +1,23 @@ +/* +Copyright 2020 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +// Imports and register the gRPC vtctl client. + +import ( + _ "vitess.io/vitess/go/vt/vtctl/grpcvtctlclient" +) From 4352126720f055f31838f96d8cb3f9a1de154589 Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Wed, 2 Jun 2021 19:52:42 +0530 Subject: [PATCH 04/64] nil srvVschema test Signed-off-by: Harshit Gangal --- go/vt/vtgate/vschema_manager.go | 16 +++++++--------- go/vt/vtgate/vschema_manager_test.go | 28 ++++++++++++++++++++++++---- 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/go/vt/vtgate/vschema_manager.go b/go/vt/vtgate/vschema_manager.go index 1d1b1a221c..2db5dc691b 100644 --- a/go/vt/vtgate/vschema_manager.go +++ b/go/vt/vtgate/vschema_manager.go @@ -159,15 +159,13 @@ func (vm *VSchemaManager) Rebuild() { var err error if v == nil { - // We encountered an error, we should always have a current vschema - log.Warning("got a schema changed signal with no loaded vschema. if this persist, something is wrong") - vschema, _ = vindexes.BuildVSchema(&vschemapb.SrvVSchema{}) - } else { - vschema, err = vm.buildAndEnhanceVSchema(v) - if err != nil { - log.Error("failed to reload vschema after schema change") - return - } + v = &vschemapb.SrvVSchema{} + } + + vschema, err = vm.buildAndEnhanceVSchema(v) + if err != nil { + log.Error("failed to reload vschema after schema change") + return } if vm.subscriber != nil { diff --git a/go/vt/vtgate/vschema_manager_test.go b/go/vt/vtgate/vschema_manager_test.go index 24fbac77f4..976889907e 100644 --- a/go/vt/vtgate/vschema_manager_test.go +++ b/go/vt/vtgate/vschema_manager_test.go @@ -84,6 +84,18 @@ func TestWatchSrvVSchema(t *testing.T) { ColumnListAuthoritative: true, }, }, + }, { + name: "empty srvVschema - schema change still applied", + schema: map[string][]vindexes.Column{"tbl": cols}, + expected: map[string]*vindexes.Table{ + "dual": dual, + "tbl": { + Name: sqlparser.NewTableIdent("tbl"), + Keyspace: ks, + Columns: cols2, + ColumnListAuthoritative: true, + }, + }, }} vm := &VSchemaManager{} @@ -99,8 +111,12 @@ func TestWatchSrvVSchema(t *testing.T) { require.NotNil(t, vs) ks := vs.Keyspaces["ks"] - require.NotNil(t, ks, "keyspace was not found") - utils.MustMatch(t, tcase.expected, ks.Tables) + if tcase.srvVschema != nil { + require.NotNil(t, ks, "keyspace was not found") + utils.MustMatch(t, tcase.expected, ks.Tables) + } else { + require.Nil(t, ks, "keyspace found") + } }) t.Run("Schema updated - "+tcase.name, func(t *testing.T) { vs = nil @@ -110,8 +126,12 @@ func TestWatchSrvVSchema(t *testing.T) { require.NotNil(t, vs) ks := vs.Keyspaces["ks"] - require.NotNil(t, ks, "keyspace was not found") - utils.MustMatch(t, tcase.expected, ks.Tables) + if tcase.srvVschema != nil { + require.NotNil(t, ks, "keyspace was not found") + utils.MustMatch(t, tcase.expected, ks.Tables) + } else { + require.Nil(t, ks, "keyspace found") + } }) } } From a2572e2b292ed31fa91aef2eda780073b9ff150a Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Fri, 7 May 2021 14:18:19 +0200 Subject: [PATCH 05/64] Add additional comparison operators in Materialize Filters and fix bug where comparison operators were not applied for sharded keyspaces Signed-off-by: Rohit Nayak --- .../endtoend/vreplication/materialize_test.go | 94 +++++++++++++++++++ .../tabletserver/vstreamer/planbuilder.go | 74 ++++++++++++--- .../vstreamer/planbuilder_test.go | 79 ++++++++++++++++ .../tabletserver/vstreamer/vstreamer.go | 2 +- go/vt/wrangler/materializer.go | 27 ++++-- 5 files changed, 256 insertions(+), 20 deletions(-) create mode 100644 go/test/endtoend/vreplication/materialize_test.go diff --git a/go/test/endtoend/vreplication/materialize_test.go b/go/test/endtoend/vreplication/materialize_test.go new file mode 100644 index 0000000000..d9cd64a261 --- /dev/null +++ b/go/test/endtoend/vreplication/materialize_test.go @@ -0,0 +1,94 @@ +/* +Copyright 2020 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vreplication + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +const smSchema = ` + CREATE TABLE tx ( + id bigint NOT NULL, + val varbinary(10) NOT NULL, + ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + typ tinyint NOT NULL, + PRIMARY KEY (id), + KEY ts (ts), + KEY typ (typ) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +` + +const smVSchema = ` +{ + "sharded": true, + "tables": { + "tx": { + "column_vindexes": [ + { + "column": "id", + "name": "hash" + } + ] + } + }, + "vindexes": { + "hash": { + "type": "hash" + } + } +} +` + +const smMaterializeSpec = `{"workflow": "wf1", "source_keyspace": "ks1", "target_keyspace": "ks2", "table_settings": [ {"target_table": "tx", "source_expression": "select * from tx where typ>=2 and val > 'abc'" }] }` + +const initDataQuery = `insert into ks1.tx(id, typ, val) values (1, 1, 'abc'), (2, 1, 'def'), (3, 2, 'def'), (4, 2, 'abc'), (5, 3, 'def'), (6, 3, 'abc')` + +func TestShardedMaterialize(t *testing.T) { + defaultCellName := "zone1" + allCells := []string{"zone1"} + allCellNames = "zone1" + vc = NewVitessCluster(t, "TestShardedMaterialize", allCells, mainClusterConfig) + ks1 := "ks1" + ks2 := "ks2" + require.NotNil(t, vc) + defaultReplicas = 0 // because of CI resource constraints we can only run this test with master tablets + defer func() { defaultReplicas = 1 }() + + //defer vc.TearDown(t) + + defaultCell = vc.Cells[defaultCellName] + vc.AddKeyspace(t, []*Cell{defaultCell}, ks1, "-", smVSchema, smSchema, defaultReplicas, defaultRdonly, 100) + vtgate = defaultCell.Vtgates[0] + require.NotNil(t, vtgate) + vtgate.WaitForStatusOfTabletInShard(fmt.Sprintf("%s.%s.master", ks1, "0"), 1) + + vc.AddKeyspace(t, []*Cell{defaultCell}, ks2, "-", smVSchema, smSchema, defaultReplicas, defaultRdonly, 200) + vtgate.WaitForStatusOfTabletInShard(fmt.Sprintf("%s.%s.master", ks2, "0"), 1) + + vtgateConn = getConnection(t, vc.ClusterConfig.hostname, vc.ClusterConfig.vtgateMySQLPort) + defer vtgateConn.Close() + verifyClusterHealth(t, vc) + _, err := vtgateConn.ExecuteFetch(initDataQuery, 0, false) + require.NoError(t, err) + materialize(t, smMaterializeSpec) + time.Sleep(5 * time.Second) + validateCount(t, vtgateConn, ks2, "tx", 2) +} diff --git a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go index eed93df110..883727493f 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go +++ b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go @@ -22,13 +22,13 @@ import ( "strconv" "strings" + "vitess.io/vitess/go/vt/vtgate/evalengine" + vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" "vitess.io/vitess/go/vt/vterrors" "vitess.io/vitess/go/vt/log" - "vitess.io/vitess/go/vt/vtgate/evalengine" - "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/key" @@ -57,10 +57,20 @@ type Plan struct { type Opcode int const ( - // Equal is used to filter an integer column on a specific value + // Equal is used to filter a comparable column on a specific value Equal = Opcode(iota) // VindexMatch is used for an in_keyrange() construct VindexMatch + // LessThan is used to filter a comparable column if < specific value + LessThan + // LessThanEqual is used to filter a comparable column if <= specific value + LessThanEqual + // GreaterThan is used to filter a comparable column if > specific value + GreaterThan + // GreaterThanEqual is used to filter a comparable column if >= specific value + GreaterThanEqual + // NotEqual is used to filter a comparable column if != specific value + NotEqual ) // Filter contains opcodes for filtering. @@ -111,6 +121,42 @@ func (plan *Plan) fields() []*querypb.Field { return fields } +// getOpcode returns the equivalent planbuilder opcode for operators that are supported in Filters +func getOpcode(comparison *sqlparser.ComparisonExpr) (Opcode, error) { + var opcode Opcode + switch comparison.Operator { + case sqlparser.EqualOp: + opcode = Equal + case sqlparser.LessThanOp: + opcode = LessThan + case sqlparser.LessEqualOp: + opcode = LessThanEqual + case sqlparser.GreaterThanOp: + opcode = GreaterThan + case sqlparser.GreaterEqualOp: + opcode = GreaterThanEqual + case sqlparser.NotEqualOp: + opcode = NotEqual + default: + return -1, fmt.Errorf("comparison operator %s not supported", comparison.Operator.ToString()) + } + return opcode, nil +} + +// compare returns true after applying the comparison specified in the Filter to the actual data in the column +func compare(comparison Opcode, columnValue, filterValue sqltypes.Value) (bool, error) { + result, err := evalengine.NullsafeCompare(columnValue, filterValue) + if err != nil { + return false, err + } + if result == 0 && (comparison == Equal || comparison == LessThanEqual || comparison == GreaterThanEqual) || + result < 0 && (comparison == LessThan || comparison == LessThanEqual || comparison == NotEqual) || + result > 0 && (comparison == GreaterThan || comparison == GreaterThanEqual || comparison == NotEqual) { + return true, nil + } + return false, nil +} + // filter filters the row against the plan. It returns false if the row did not match. // The output of the filtering operation is stored in the 'result' argument because // filtering cannot be performed in-place. The result argument must be a slice of @@ -121,14 +167,6 @@ func (plan *Plan) filter(values, result []sqltypes.Value) (bool, error) { } for _, filter := range plan.Filters { switch filter.Opcode { - case Equal: - result, err := evalengine.NullsafeCompare(values[filter.ColNum], filter.Value) - if err != nil { - return false, err - } - if result != 0 { - return false, nil - } case VindexMatch: ksid, err := getKeyspaceID(values, filter.Vindex, filter.VindexColumns, plan.Table.Fields) if err != nil { @@ -137,6 +175,14 @@ func (plan *Plan) filter(values, result []sqltypes.Value) (bool, error) { if !key.KeyRangeContains(filter.KeyRange, ksid) { return false, nil } + default: + match, err := compare(filter.Opcode, values[filter.ColNum], filter.Value) + if err != nil { + return false, err + } + if !match { + return false, nil + } } } for i, colExpr := range plan.ColExprs { @@ -372,6 +418,10 @@ func (plan *Plan) analyzeWhere(vschema *localVSchema, where *sqlparser.Where) er for _, expr := range exprs { switch expr := expr.(type) { case *sqlparser.ComparisonExpr: + opcode, err := getOpcode(expr) + if err != nil { + return err + } qualifiedName, ok := expr.Left.(*sqlparser.ColName) if !ok { return fmt.Errorf("unexpected: %v", sqlparser.String(expr)) @@ -400,7 +450,7 @@ func (plan *Plan) analyzeWhere(vschema *localVSchema, where *sqlparser.Where) er return err } plan.Filters = append(plan.Filters, Filter{ - Opcode: Equal, + Opcode: opcode, ColNum: colnum, Value: resolved, }) diff --git a/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go b/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go index 35c1e5d41f..d0269a4468 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go @@ -20,6 +20,8 @@ import ( "fmt" "testing" + "vitess.io/vitess/go/vt/proto/topodata" + "github.com/stretchr/testify/require" "vitess.io/vitess/go/test/utils" @@ -585,3 +587,80 @@ func TestPlanBuilder(t *testing.T) { }) } } + +func TestPlanBuilderFilterComparison(t *testing.T) { + t1 := &Table{ + Name: "t1", + Fields: []*querypb.Field{{ + Name: "id", + Type: sqltypes.Int64, + }, { + Name: "val", + Type: sqltypes.VarBinary, + }}, + } + hashVindex, err := vindexes.NewHash("hash", nil) + require.NoError(t, err) + testcases := []struct { + name string + inFilter string + outFilters []Filter + outErr string + }{{ + name: "equal", + inFilter: "select * from t1 where id = 1", + outFilters: []Filter{{Opcode: Equal, ColNum: 0, Value: sqltypes.NewInt64(1)}}, + }, { + name: "not-equal", + inFilter: "select * from t1 where id <> 1", + outFilters: []Filter{{Opcode: NotEqual, ColNum: 0, Value: sqltypes.NewInt64(1)}}, + }, { + name: "greater", + inFilter: "select * from t1 where val > 'abc'", + outFilters: []Filter{{Opcode: GreaterThan, ColNum: 1, Value: sqltypes.NewVarBinary("abc")}}, + }, { + name: "greater-than", + inFilter: "select * from t1 where id >= 1", + outFilters: []Filter{{Opcode: GreaterThanEqual, ColNum: 0, Value: sqltypes.NewInt64(1)}}, + }, { + name: "less-than-with-and", + inFilter: "select * from t1 where id < 2 and val <= 'xyz'", + outFilters: []Filter{{Opcode: LessThan, ColNum: 0, Value: sqltypes.NewInt64(2)}, + {Opcode: LessThanEqual, ColNum: 1, Value: sqltypes.NewVarBinary("xyz")}, + }, + }, { + name: "vindex-and-operators", + inFilter: "select * from t1 where in_keyrange(id, 'hash', '-80') and id = 2 and val <> 'xyz'", + outFilters: []Filter{ + { + Opcode: VindexMatch, + ColNum: 0, + Value: sqltypes.NULL, + Vindex: hashVindex, + VindexColumns: []int{0}, + KeyRange: &topodata.KeyRange{ + Start: nil, + End: []byte("\200"), + }, + }, + {Opcode: Equal, ColNum: 0, Value: sqltypes.NewInt64(2)}, + {Opcode: NotEqual, ColNum: 1, Value: sqltypes.NewVarBinary("xyz")}, + }, + }} + + for _, tcase := range testcases { + t.Run(tcase.name, func(t *testing.T) { + plan, err := buildPlan(t1, testLocalVSchema, &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{Match: "t1", Filter: tcase.inFilter}}, + }) + + if tcase.outErr != "" { + assert.Nil(t, plan) + assert.EqualError(t, err, tcase.outErr) + return + } + require.NotNil(t, plan) + require.ElementsMatchf(t, tcase.outFilters, plan.Filters, "want %+v, got: %+v", tcase.outFilters, plan.Filters) + }) + } +} diff --git a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go index 65347a2cca..e737c3211e 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go +++ b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go @@ -102,7 +102,7 @@ type streamerPlan struct { // "select * from t where in_keyrange(col1, 'hash', '-80')", // "select col1, col2 from t where...", // "select col1, keyspace_id() from t where...". -// Only "in_keyrange" expressions are supported in the where clause. +// Only "in_keyrange" and limited comparison operators (see enum Opcode in planbuilder.go) are supported in the where clause. // Other constructs like joins, group by, etc. are not supported. // vschema: the current vschema. This value can later be changed through the SetVSchema method. // send: callback function to send events. diff --git a/go/vt/wrangler/materializer.go b/go/vt/wrangler/materializer.go index 163e300ea9..1460076131 100644 --- a/go/vt/wrangler/materializer.go +++ b/go/vt/wrangler/materializer.go @@ -1032,7 +1032,6 @@ func (mz *materializer) generateInserts(ctx context.Context) (string, error) { if !ok { return "", fmt.Errorf("unrecognized statement: %s", ts.SourceExpression) } - filter := ts.SourceExpression if mz.targetVSchema.Keyspace.Sharded && mz.targetVSchema.Tables[ts.TargetTable].Type != vindexes.TypeReference { cv, err := vindexes.FindBestColVindex(mz.targetVSchema.Tables[ts.TargetTable]) @@ -1054,12 +1053,26 @@ func (mz *materializer) generateInserts(ctx context.Context) (string, error) { vindexName := fmt.Sprintf("%s.%s", mz.ms.TargetKeyspace, cv.Name) subExprs = append(subExprs, &sqlparser.AliasedExpr{Expr: sqlparser.NewStrLiteral(vindexName)}) subExprs = append(subExprs, &sqlparser.AliasedExpr{Expr: sqlparser.NewStrLiteral("{{.keyrange}}")}) - sel.Where = &sqlparser.Where{ - Type: sqlparser.WhereClause, - Expr: &sqlparser.FuncExpr{ - Name: sqlparser.NewColIdent("in_keyrange"), - Exprs: subExprs, - }, + inKeyRange := &sqlparser.FuncExpr{ + Name: sqlparser.NewColIdent("in_keyrange"), + Exprs: subExprs, + } + if sel.Where != nil { + sel.Where = &sqlparser.Where{ + Type: sqlparser.WhereClause, + Expr: &sqlparser.AndExpr{ + Left: inKeyRange, + Right: sel.Where.Expr, + }, + } + } else { + sel.Where = &sqlparser.Where{ + Type: sqlparser.WhereClause, + Expr: &sqlparser.FuncExpr{ + Name: sqlparser.NewColIdent("in_keyrange"), + Exprs: subExprs, + }, + } } filter = sqlparser.String(sel) From 22117f32369b1f49b3f6e780042be10ca1f6ae90 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Fri, 7 May 2021 15:55:56 +0200 Subject: [PATCH 06/64] Update materialize e2e test and add to config.json Signed-off-by: Rohit Nayak --- go/test/endtoend/vreplication/materialize_test.go | 12 ++++++++---- test/config.json | 9 +++++++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/go/test/endtoend/vreplication/materialize_test.go b/go/test/endtoend/vreplication/materialize_test.go index d9cd64a261..0f576d9405 100644 --- a/go/test/endtoend/vreplication/materialize_test.go +++ b/go/test/endtoend/vreplication/materialize_test.go @@ -1,5 +1,5 @@ /* -Copyright 2020 The Vitess Authors. +Copyright 2021 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -19,7 +19,6 @@ package vreplication import ( "fmt" "testing" - "time" "github.com/stretchr/testify/require" ) @@ -61,6 +60,7 @@ const smMaterializeSpec = `{"workflow": "wf1", "source_keyspace": "ks1", "target const initDataQuery = `insert into ks1.tx(id, typ, val) values (1, 1, 'abc'), (2, 1, 'def'), (3, 2, 'def'), (4, 2, 'abc'), (5, 3, 'def'), (6, 3, 'abc')` +// TestShardedMaterialize tests a materialize from a sharded (single shard) using comparison filters func TestShardedMaterialize(t *testing.T) { defaultCellName := "zone1" allCells := []string{"zone1"} @@ -72,7 +72,7 @@ func TestShardedMaterialize(t *testing.T) { defaultReplicas = 0 // because of CI resource constraints we can only run this test with master tablets defer func() { defaultReplicas = 1 }() - //defer vc.TearDown(t) + defer vc.TearDown(t) defaultCell = vc.Cells[defaultCellName] vc.AddKeyspace(t, []*Cell{defaultCell}, ks1, "-", smVSchema, smSchema, defaultReplicas, defaultRdonly, 100) @@ -89,6 +89,10 @@ func TestShardedMaterialize(t *testing.T) { _, err := vtgateConn.ExecuteFetch(initDataQuery, 0, false) require.NoError(t, err) materialize(t, smMaterializeSpec) - time.Sleep(5 * time.Second) + tab := vc.Cells[defaultCell.Name].Keyspaces[ks2].Shards["-"].Tablets["zone1-200"].Vttablet + catchup(t, tab, "wf1", "Materialize") + validateCount(t, vtgateConn, ks2, "tx", 2) + validateQuery(t, vtgateConn, "ks2:-", "select id, val from tx", + `[[INT64(3) VARBINARY("def")] [INT64(5) VARBINARY("def")]]`) } diff --git a/test/config.json b/test/config.json index 430c42a421..4847136742 100644 --- a/test/config.json +++ b/test/config.json @@ -752,6 +752,15 @@ "RetryMax": 0, "Tags": [] }, + "vreplication_materialize": { + "File": "unused.go", + "Args": ["vitess.io/vitess/go/test/endtoend/vreplication", "-run", "ShardedMaterialize"], + "Command": [], + "Manual": false, + "Shard": "vreplication_multicell", + "RetryMax": 0, + "Tags": [] + }, "vreplication_cellalias": { "File": "unused.go", "Args": ["vitess.io/vitess/go/test/endtoend/vreplication", "-run", "CellAlias"], From 1f8cc8fc50840b37163385f2e9a29d03c60b6db6 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Tue, 11 May 2021 21:20:56 +0200 Subject: [PATCH 07/64] Refactor comparison based on review comments Signed-off-by: Rohit Nayak --- .../tabletserver/vstreamer/planbuilder.go | 33 ++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go index 883727493f..c6bb64fbb8 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go +++ b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go @@ -145,14 +145,39 @@ func getOpcode(comparison *sqlparser.ComparisonExpr) (Opcode, error) { // compare returns true after applying the comparison specified in the Filter to the actual data in the column func compare(comparison Opcode, columnValue, filterValue sqltypes.Value) (bool, error) { + // NullsafeCompare returns 0 if values match, -1 if columnValue < filterValue, 1 if columnValue > filterValue result, err := evalengine.NullsafeCompare(columnValue, filterValue) if err != nil { return false, err } - if result == 0 && (comparison == Equal || comparison == LessThanEqual || comparison == GreaterThanEqual) || - result < 0 && (comparison == LessThan || comparison == LessThanEqual || comparison == NotEqual) || - result > 0 && (comparison == GreaterThan || comparison == GreaterThanEqual || comparison == NotEqual) { - return true, nil + + switch comparison { + case Equal: + if result == 0 { + return true, nil + } + case NotEqual: + if result != 0 { + return true, nil + } + case LessThan: + if result == -1 { + return true, nil + } + case LessThanEqual: + if result <= 0 { + return true, nil + } + case GreaterThan: + if result == 1 { + return true, nil + } + case GreaterThanEqual: + if result >= 0 { + return true, nil + } + default: + return false, fmt.Errorf("comparison operator %d not supported", comparison) } return false, nil } From 2a42a841f8941260f6f9173988e6ea6a4535ad18 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Wed, 2 Jun 2021 21:28:53 +0200 Subject: [PATCH 08/64] Add null semantics to compare and related test Signed-off-by: Rohit Nayak --- .../tabletserver/vstreamer/planbuilder.go | 5 +++ .../vstreamer/planbuilder_test.go | 36 +++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go index c6bb64fbb8..33c5319cd4 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go +++ b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go @@ -145,6 +145,11 @@ func getOpcode(comparison *sqlparser.ComparisonExpr) (Opcode, error) { // compare returns true after applying the comparison specified in the Filter to the actual data in the column func compare(comparison Opcode, columnValue, filterValue sqltypes.Value) (bool, error) { + // use null semantics: return false if either value is null + if columnValue.IsNull() || filterValue.IsNull() { + return false, nil + } + // at this point neither values can be null // NullsafeCompare returns 0 if values match, -1 if columnValue < filterValue, 1 if columnValue > filterValue result, err := evalengine.NullsafeCompare(columnValue, filterValue) if err != nil { diff --git a/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go b/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go index d0269a4468..28a96b8a24 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go @@ -664,3 +664,39 @@ func TestPlanBuilderFilterComparison(t *testing.T) { }) } } + +func TestCompare(t *testing.T) { + type testcase struct { + opcode Opcode + columnValue, filterValue sqltypes.Value + want bool + } + int1 := sqltypes.NewInt32(1) + int2 := sqltypes.NewInt32(2) + testcases := []*testcase{ + {opcode: Equal, columnValue: int1, filterValue: int1, want: true}, + {opcode: Equal, columnValue: int1, filterValue: int2, want: false}, + {opcode: Equal, columnValue: int1, filterValue: sqltypes.NULL, want: false}, + {opcode: LessThan, columnValue: int2, filterValue: int1, want: false}, + {opcode: LessThan, columnValue: int1, filterValue: int2, want: true}, + {opcode: LessThan, columnValue: int1, filterValue: sqltypes.NULL, want: false}, + {opcode: GreaterThan, columnValue: int2, filterValue: int1, want: true}, + {opcode: GreaterThan, columnValue: int1, filterValue: int2, want: false}, + {opcode: GreaterThan, columnValue: int1, filterValue: sqltypes.NULL, want: false}, + {opcode: NotEqual, columnValue: int1, filterValue: int1, want: false}, + {opcode: NotEqual, columnValue: int1, filterValue: int2, want: true}, + {opcode: NotEqual, columnValue: sqltypes.NULL, filterValue: int1, want: false}, + {opcode: LessThanEqual, columnValue: int1, filterValue: sqltypes.NULL, want: false}, + {opcode: GreaterThanEqual, columnValue: int2, filterValue: int1, want: true}, + {opcode: LessThanEqual, columnValue: int2, filterValue: int1, want: false}, + {opcode: GreaterThanEqual, columnValue: int1, filterValue: int1, want: true}, + {opcode: LessThanEqual, columnValue: int1, filterValue: int2, want: true}, + } + for _, tc := range testcases { + t.Run("", func(t *testing.T) { + got, err := compare(tc.opcode, tc.columnValue, tc.filterValue) + require.NoError(t, err) + require.Equal(t, tc.want, got) + }) + } +} From 12a456e8444e608129a679cd61985c02cf67c33c Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Mon, 31 May 2021 13:38:12 +0530 Subject: [PATCH 09/64] added function to create vttestserver docker images Signed-off-by: GuptaManan100 --- go/test/endtoend/docker/vttestserver_test.go | 75 ++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 go/test/endtoend/docker/vttestserver_test.go diff --git a/go/test/endtoend/docker/vttestserver_test.go b/go/test/endtoend/docker/vttestserver_test.go new file mode 100644 index 0000000000..5e74a998e4 --- /dev/null +++ b/go/test/endtoend/docker/vttestserver_test.go @@ -0,0 +1,75 @@ +/* +Copyright 2021 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package docker + +import ( + "os" + "os/exec" + "path" + "testing" + + "github.com/golang/glog" +) + +const ( + vttestserverMysql57image = "vttestserver-e2etest/mysql57" + vttestserverMysql80image = "vttestserver-e2etest/mysql80" +) + +func TestMain(m *testing.M) { + exitCode := func() int { + err := makeVttestserverDockerImages() + if err != nil { + glog.Error(err.Error()) + return 1 + } + return m.Run() + }() + os.Exit(exitCode) +} + +//makeVttestserverDockerImages creates the vttestserver docker images for both MySQL57 and MySQL80 +func makeVttestserverDockerImages() error { + mainVitessPath := path.Join(os.Getenv("PWD"), "../../../..") + dockerFilePath := path.Join(mainVitessPath, "docker/vttestserver/Dockerfile.mysql57") + cmd57 := exec.Command("docker", "build", "-f", dockerFilePath, "-t", vttestserverMysql57image, ".") + cmd57.Dir = mainVitessPath + err := cmd57.Start() + if err != nil { + return err + } + + dockerFilePath = path.Join(mainVitessPath, "docker/vttestserver/Dockerfile.mysql80") + cmd80 := exec.Command("docker", "build", "-f", dockerFilePath, "-t", vttestserverMysql80image, ".") + cmd80.Dir = mainVitessPath + err = cmd80.Start() + if err != nil { + return err + } + + err = cmd57.Wait() + if err != nil { + return err + } + + err = cmd80.Wait() + if err != nil { + return err + } + + return nil +} From c425e9a6eb94f25b645078de545170ef8cc90ca5 Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Mon, 31 May 2021 16:35:55 +0530 Subject: [PATCH 10/64] added test for unsharded keyspace in vttestserver image Signed-off-by: GuptaManan100 --- go/test/endtoend/docker/vttestserver.go | 119 +++++++++++++++++++ go/test/endtoend/docker/vttestserver_test.go | 91 ++++++++------ 2 files changed, 175 insertions(+), 35 deletions(-) create mode 100644 go/test/endtoend/docker/vttestserver.go diff --git a/go/test/endtoend/docker/vttestserver.go b/go/test/endtoend/docker/vttestserver.go new file mode 100644 index 0000000000..81f9266148 --- /dev/null +++ b/go/test/endtoend/docker/vttestserver.go @@ -0,0 +1,119 @@ +/* +Copyright 2021 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package docker + +import ( + "fmt" + "os" + "os/exec" + "path" + "strconv" + "strings" +) + +const ( + vttestserverMysql57image = "vttestserver-e2etest/mysql57" + vttestserverMysql80image = "vttestserver-e2etest/mysql80" +) + +type vttestserver struct { + dockerImage string + keyspaces []string + numShards []int + mysqlMaxConnecetions int + port int +} + +func newVttestserver(dockerImage string, keyspaces []string, numShards []int, mysqlMaxConnections, port int) *vttestserver { + return &vttestserver{ + dockerImage: dockerImage, + keyspaces: keyspaces, + numShards: numShards, + mysqlMaxConnecetions: mysqlMaxConnections, + port: port, + } +} + +func (v *vttestserver) teardown() { + cmd := exec.Command("docker", "rm", "--force", "vttestserver-end2end-test") + _ = cmd.Run() +} + +// startDockerImage starts the docker image for the vttestserver +func (v *vttestserver) startDockerImage() error { + cmd := exec.Command("docker", "run") + cmd.Args = append(cmd.Args, "--name=vttestserver-end2end-test") + cmd.Args = append(cmd.Args, "-p", fmt.Sprintf("%d:33577", v.port)) + cmd.Args = append(cmd.Args, "-e", "PORT=33574") + cmd.Args = append(cmd.Args, "-e", fmt.Sprintf("KEYSPACES=%s", strings.Join(v.keyspaces, ","))) + cmd.Args = append(cmd.Args, "-e", fmt.Sprintf("NUM_SHARDS=%s", strings.Join(convertToStringSlice(v.numShards), ","))) + cmd.Args = append(cmd.Args, "-e", "MYSQL_BIND_HOST=0.0.0.0") + cmd.Args = append(cmd.Args, "-e", fmt.Sprintf("MYSQL_MAX_CONNECTIONS=%d", v.mysqlMaxConnecetions)) + cmd.Args = append(cmd.Args, `--health-cmd="mysqladmin ping -h127.0.0.1 -P33577"`) + cmd.Args = append(cmd.Args, "--health-interval=5s") + cmd.Args = append(cmd.Args, "--health-timeout=2s") + cmd.Args = append(cmd.Args, "--health-retries=5") + cmd.Args = append(cmd.Args, v.dockerImage) + + err := cmd.Start() + if err != nil { + return err + } + return nil +} + +// convertToStringSlice converts an integer slice to string slice +func convertToStringSlice(intSlice []int) []string { + var stringSlice []string + for _, val := range intSlice { + str := strconv.Itoa(val) + stringSlice = append(stringSlice, str) + } + return stringSlice +} + +//makeVttestserverDockerImages creates the vttestserver docker images for both MySQL57 and MySQL80 +func makeVttestserverDockerImages() error { + mainVitessPath := path.Join(os.Getenv("PWD"), "../../../..") + dockerFilePath := path.Join(mainVitessPath, "docker/vttestserver/Dockerfile.mysql57") + cmd57 := exec.Command("docker", "build", "-f", dockerFilePath, "-t", vttestserverMysql57image, ".") + cmd57.Dir = mainVitessPath + err := cmd57.Start() + if err != nil { + return err + } + + dockerFilePath = path.Join(mainVitessPath, "docker/vttestserver/Dockerfile.mysql80") + cmd80 := exec.Command("docker", "build", "-f", dockerFilePath, "-t", vttestserverMysql80image, ".") + cmd80.Dir = mainVitessPath + err = cmd80.Start() + if err != nil { + return err + } + + err = cmd57.Wait() + if err != nil { + return err + } + + err = cmd80.Wait() + if err != nil { + return err + } + + return nil +} diff --git a/go/test/endtoend/docker/vttestserver_test.go b/go/test/endtoend/docker/vttestserver_test.go index 5e74a998e4..ad3a85d90c 100644 --- a/go/test/endtoend/docker/vttestserver_test.go +++ b/go/test/endtoend/docker/vttestserver_test.go @@ -17,24 +17,25 @@ limitations under the License. package docker import ( + "context" + "fmt" "os" - "os/exec" - "path" "testing" + "time" - "github.com/golang/glog" -) + "github.com/google/go-cmp/cmp" -const ( - vttestserverMysql57image = "vttestserver-e2etest/mysql57" - vttestserverMysql80image = "vttestserver-e2etest/mysql80" + "vitess.io/vitess/go/sqltypes" + + "vitess.io/vitess/go/mysql" + + "github.com/stretchr/testify/require" ) func TestMain(m *testing.M) { exitCode := func() int { err := makeVttestserverDockerImages() if err != nil { - glog.Error(err.Error()) return 1 } return m.Run() @@ -42,34 +43,54 @@ func TestMain(m *testing.M) { os.Exit(exitCode) } -//makeVttestserverDockerImages creates the vttestserver docker images for both MySQL57 and MySQL80 -func makeVttestserverDockerImages() error { - mainVitessPath := path.Join(os.Getenv("PWD"), "../../../..") - dockerFilePath := path.Join(mainVitessPath, "docker/vttestserver/Dockerfile.mysql57") - cmd57 := exec.Command("docker", "build", "-f", dockerFilePath, "-t", vttestserverMysql57image, ".") - cmd57.Dir = mainVitessPath - err := cmd57.Start() - if err != nil { - return err - } +func TestUnsharded(t *testing.T) { + dockerImages := []string{vttestserverMysql57image, vttestserverMysql80image} + for _, image := range dockerImages { + t.Run(image, func(t *testing.T) { + vtest := newVttestserver(image, []string{"unsharded_ks"}, []int{1}, 1000, 33577) + err := vtest.startDockerImage() + require.NoError(t, err) + defer vtest.teardown() - dockerFilePath = path.Join(mainVitessPath, "docker/vttestserver/Dockerfile.mysql80") - cmd80 := exec.Command("docker", "build", "-f", dockerFilePath, "-t", vttestserverMysql80image, ".") - cmd80.Dir = mainVitessPath - err = cmd80.Start() - if err != nil { - return err - } + // wait for the docker to be setup + time.Sleep(10 * time.Second) - err = cmd57.Wait() - if err != nil { - return err + ctx := context.Background() + vttestParams := mysql.ConnParams{ + Host: "localhost", + Port: vtest.port, + } + conn, err := mysql.Connect(ctx, &vttestParams) + require.NoError(t, err) + defer conn.Close() + assertMatches(t, conn, "show databases", `[[VARCHAR("unsharded_ks")] [VARCHAR("information_schema")] [VARCHAR("mysql")] [VARCHAR("sys")] [VARCHAR("performance_schema")]]`) + _, err = execute(t, conn, "create table unsharded_ks.t1(id int)") + require.NoError(t, err) + _, err = execute(t, conn, "insert into unsharded_ks.t1(id) values (10),(20),(30)") + require.NoError(t, err) + assertMatches(t, conn, "select * from unsharded_ks.t1", `[[INT32(10)] [INT32(20)] [INT32(30)]]`) + }) + } +} + +func execute(t *testing.T, conn *mysql.Conn, query string) (*sqltypes.Result, error) { + t.Helper() + return conn.ExecuteFetch(query, 1000, true) +} + +func checkedExec(t *testing.T, conn *mysql.Conn, query string) *sqltypes.Result { + t.Helper() + qr, err := conn.ExecuteFetch(query, 1000, true) + require.NoError(t, err) + return qr +} + +func assertMatches(t *testing.T, conn *mysql.Conn, query, expected string) { + t.Helper() + qr := checkedExec(t, conn, query) + got := fmt.Sprintf("%v", qr.Rows) + diff := cmp.Diff(expected, got) + if diff != "" { + t.Errorf("Query: %s (-want +got):\n%s", query, diff) } - - err = cmd80.Wait() - if err != nil { - return err - } - - return nil } From 0a3baf882341dde3dc531573bac3faabe2db1d3e Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Mon, 31 May 2021 16:45:05 +0530 Subject: [PATCH 11/64] added test for sharded keyspace for vttestserver image Signed-off-by: GuptaManan100 --- go/test/endtoend/docker/vttestserver_test.go | 32 ++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/go/test/endtoend/docker/vttestserver_test.go b/go/test/endtoend/docker/vttestserver_test.go index ad3a85d90c..12a3f00329 100644 --- a/go/test/endtoend/docker/vttestserver_test.go +++ b/go/test/endtoend/docker/vttestserver_test.go @@ -73,6 +73,38 @@ func TestUnsharded(t *testing.T) { } } +func TestSharded(t *testing.T) { + dockerImages := []string{vttestserverMysql57image, vttestserverMysql80image} + for _, image := range dockerImages { + t.Run(image, func(t *testing.T) { + vtest := newVttestserver(image, []string{"ks"}, []int{2}, 1000, 33577) + err := vtest.startDockerImage() + require.NoError(t, err) + defer vtest.teardown() + + // wait for the docker to be setup + time.Sleep(10 * time.Second) + + ctx := context.Background() + vttestParams := mysql.ConnParams{ + Host: "localhost", + Port: vtest.port, + } + conn, err := mysql.Connect(ctx, &vttestParams) + require.NoError(t, err) + defer conn.Close() + assertMatches(t, conn, "show databases", `[[VARCHAR("ks")] [VARCHAR("information_schema")] [VARCHAR("mysql")] [VARCHAR("sys")] [VARCHAR("performance_schema")]]`) + _, err = execute(t, conn, "create table ks.t1(id int)") + require.NoError(t, err) + _, err = execute(t, conn, "alter vschema on ks.t1 add vindex `binary_md5`(id) using `binary_md5`") + require.NoError(t, err) + _, err = execute(t, conn, "insert into ks.t1(id) values (10),(20),(30)") + require.NoError(t, err) + assertMatches(t, conn, "select id from ks.t1 order by id", `[[INT32(10)] [INT32(20)] [INT32(30)]]`) + }) + } +} + func execute(t *testing.T, conn *mysql.Conn, query string) (*sqltypes.Result, error) { t.Helper() return conn.ExecuteFetch(query, 1000, true) From f01e28939e2eaffe8c07146c90ebe115244b9940 Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Mon, 31 May 2021 16:48:11 +0530 Subject: [PATCH 12/64] added test for mysql max connections for vttestserver image Signed-off-by: GuptaManan100 --- go/test/endtoend/docker/vttestserver_test.go | 25 ++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/go/test/endtoend/docker/vttestserver_test.go b/go/test/endtoend/docker/vttestserver_test.go index 12a3f00329..504cdee144 100644 --- a/go/test/endtoend/docker/vttestserver_test.go +++ b/go/test/endtoend/docker/vttestserver_test.go @@ -105,6 +105,31 @@ func TestSharded(t *testing.T) { } } +func TestMysqlMaxCons(t *testing.T) { + dockerImages := []string{vttestserverMysql57image, vttestserverMysql80image} + for _, image := range dockerImages { + t.Run(image, func(t *testing.T) { + vtest := newVttestserver(image, []string{"ks"}, []int{2}, 100000, 33577) + err := vtest.startDockerImage() + require.NoError(t, err) + defer vtest.teardown() + + // wait for the docker to be setup + time.Sleep(10 * time.Second) + + ctx := context.Background() + vttestParams := mysql.ConnParams{ + Host: "localhost", + Port: vtest.port, + } + conn, err := mysql.Connect(ctx, &vttestParams) + require.NoError(t, err) + defer conn.Close() + assertMatches(t, conn, "select @@max_connections", `[[UINT64(100000)]]`) + }) + } +} + func execute(t *testing.T, conn *mysql.Conn, query string) (*sqltypes.Result, error) { t.Helper() return conn.ExecuteFetch(query, 1000, true) From d2e8b5ab81a635e2686eef0edbfa19d09c35ed92 Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Mon, 31 May 2021 17:00:52 +0530 Subject: [PATCH 13/64] added test for lots of keyspaces for vttestserver image Signed-off-by: GuptaManan100 --- go/test/endtoend/docker/vttestserver_test.go | 40 ++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/go/test/endtoend/docker/vttestserver_test.go b/go/test/endtoend/docker/vttestserver_test.go index 504cdee144..4226f0c0c0 100644 --- a/go/test/endtoend/docker/vttestserver_test.go +++ b/go/test/endtoend/docker/vttestserver_test.go @@ -130,6 +130,46 @@ func TestMysqlMaxCons(t *testing.T) { } } +func TestLargeNumberOfKeyspaces(t *testing.T) { + dockerImages := []string{vttestserverMysql57image, vttestserverMysql80image} + for _, image := range dockerImages { + t.Run(image, func(t *testing.T) { + var keyspaces []string + var numShards []int + for i := 0; i < 100; i++ { + keyspaces = append(keyspaces, fmt.Sprintf("unsharded_ks%d", i)) + numShards = append(numShards, 1) + } + + vtest := newVttestserver(image, keyspaces, numShards, 100000, 33577) + err := vtest.startDockerImage() + require.NoError(t, err) + defer vtest.teardown() + + // wait for the docker to be setup + time.Sleep(15 * time.Second) + + ctx := context.Background() + vttestParams := mysql.ConnParams{ + Host: "localhost", + Port: vtest.port, + } + conn, err := mysql.Connect(ctx, &vttestParams) + require.NoError(t, err) + defer conn.Close() + + // assert that all the keyspaces are correctly setup + for _, keyspace := range keyspaces { + _, err = execute(t, conn, "create table "+keyspace+".t1(id int)") + require.NoError(t, err) + _, err = execute(t, conn, "insert into "+keyspace+".t1(id) values (10),(20),(30)") + require.NoError(t, err) + assertMatches(t, conn, "select * from "+keyspace+".t1", `[[INT32(10)] [INT32(20)] [INT32(30)]]`) + } + }) + } +} + func execute(t *testing.T, conn *mysql.Conn, query string) (*sqltypes.Result, error) { t.Helper() return conn.ExecuteFetch(query, 1000, true) From b8715726ec684ac3a23880e98a48fcc306de6e9e Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Thu, 3 Jun 2021 13:40:05 +0530 Subject: [PATCH 14/64] save enhanced vschema to currentVSchema and removed err from BuildVSchema Signed-off-by: Harshit Gangal --- go/vt/vtgate/engine/fuzz_flaky_test.go | 2 +- go/vt/vtgate/engine/insert_test.go | 89 ++---- go/vt/vtgate/engine/update_test.go | 5 +- go/vt/vtgate/planbuilder/plan_test.go | 2 +- go/vt/vtgate/vindexes/vschema.go | 4 +- go/vt/vtgate/vindexes/vschema_test.go | 62 ++-- go/vt/vtgate/vschema_manager.go | 39 +-- go/vt/vtgate/vschema_manager_test.go | 271 +++++++++++++----- .../vttablet/tabletserver/vstreamer/engine.go | 2 +- go/vt/vttablet/tabletserver/vstreamer/fuzz.go | 2 +- .../vstreamer/local_vschema_test.go | 10 +- .../vstreamer/planbuilder_test.go | 5 +- 12 files changed, 271 insertions(+), 222 deletions(-) diff --git a/go/vt/vtgate/engine/fuzz_flaky_test.go b/go/vt/vtgate/engine/fuzz_flaky_test.go index ef0292f5bc..538875e9a7 100644 --- a/go/vt/vtgate/engine/fuzz_flaky_test.go +++ b/go/vt/vtgate/engine/fuzz_flaky_test.go @@ -81,7 +81,7 @@ func createVSchema() (vschema *vindexes.VSchema, err error) { }, }, } - vs, err := vindexes.BuildVSchema(invschema) + vs := vindexes.BuildVSchema(invschema) if err != nil { return nil, err } diff --git a/go/vt/vtgate/engine/insert_test.go b/go/vt/vtgate/engine/insert_test.go index 0a1a628e82..d8c5d69a6a 100644 --- a/go/vt/vtgate/engine/insert_test.go +++ b/go/vt/vtgate/engine/insert_test.go @@ -195,10 +195,7 @@ func TestInsertShardedSimple(t *testing.T) { }, }, } - vs, err := vindexes.BuildVSchema(invschema) - if err != nil { - t.Fatal(err) - } + vs := vindexes.BuildVSchema(invschema) ks := vs.Keyspaces["sharded"] // A single row insert should be autocommitted @@ -222,7 +219,7 @@ func TestInsertShardedSimple(t *testing.T) { vc := newDMLTestVCursor("-20", "20-") vc.shardForKsid = []string{"20-", "-20", "20-"} - _, err = ins.Execute(vc, map[string]*querypb.BindVariable{}, false) + _, err := ins.Execute(vc, map[string]*querypb.BindVariable{}, false) if err != nil { t.Fatal(err) } @@ -342,10 +339,7 @@ func TestInsertShardedFail(t *testing.T) { }, }, } - vs, err := vindexes.BuildVSchema(invschema) - if err != nil { - t.Fatal(err) - } + vs := vindexes.BuildVSchema(invschema) ks := vs.Keyspaces["sharded"] ins := NewInsert( @@ -369,7 +363,7 @@ func TestInsertShardedFail(t *testing.T) { vc := &loggingVCursor{} // The lookup will fail to map to a keyspace id. - _, err = ins.Execute(vc, map[string]*querypb.BindVariable{}, false) + _, err := ins.Execute(vc, map[string]*querypb.BindVariable{}, false) require.EqualError(t, err, `could not map [INT64(1)] to a keyspace id`) } @@ -394,10 +388,7 @@ func TestInsertShardedGenerate(t *testing.T) { }, }, } - vs, err := vindexes.BuildVSchema(invschema) - if err != nil { - t.Fatal(err) - } + vs := vindexes.BuildVSchema(invschema) ks := vs.Keyspaces["sharded"] ins := NewInsert( @@ -519,10 +510,7 @@ func TestInsertShardedOwned(t *testing.T) { }, }, } - vs, err := vindexes.BuildVSchema(invschema) - if err != nil { - t.Fatal(err) - } + vs := vindexes.BuildVSchema(invschema) ks := vs.Keyspaces["sharded"] ins := NewInsert( @@ -583,7 +571,7 @@ func TestInsertShardedOwned(t *testing.T) { vc := newDMLTestVCursor("-20", "20-") vc.shardForKsid = []string{"20-", "-20", "20-"} - _, err = ins.Execute(vc, map[string]*querypb.BindVariable{}, false) + _, err := ins.Execute(vc, map[string]*querypb.BindVariable{}, false) if err != nil { t.Fatal(err) } @@ -646,10 +634,7 @@ func TestInsertShardedOwnedWithNull(t *testing.T) { }, }, } - vs, err := vindexes.BuildVSchema(invschema) - if err != nil { - t.Fatal(err) - } + vs := vindexes.BuildVSchema(invschema) ks := vs.Keyspaces["sharded"] ins := NewInsert( @@ -681,7 +666,7 @@ func TestInsertShardedOwnedWithNull(t *testing.T) { vc := newDMLTestVCursor("-20", "20-") vc.shardForKsid = []string{"20-", "-20", "20-"} - _, err = ins.Execute(vc, map[string]*querypb.BindVariable{}, false) + _, err := ins.Execute(vc, map[string]*querypb.BindVariable{}, false) if err != nil { t.Fatal(err) } @@ -728,10 +713,7 @@ func TestInsertShardedGeo(t *testing.T) { }, }, } - vs, err := vindexes.BuildVSchema(invschema) - if err != nil { - t.Fatal(err) - } + vs := vindexes.BuildVSchema(invschema) ks := vs.Keyspaces["sharded"] ins := NewInsert( @@ -774,7 +756,7 @@ func TestInsertShardedGeo(t *testing.T) { vc := newDMLTestVCursor("-20", "20-") vc.shardForKsid = []string{"20-", "-20"} - _, err = ins.Execute(vc, map[string]*querypb.BindVariable{}, false) + _, err := ins.Execute(vc, map[string]*querypb.BindVariable{}, false) if err != nil { t.Fatal(err) } @@ -843,10 +825,7 @@ func TestInsertShardedIgnoreOwned(t *testing.T) { }, }, } - vs, err := vindexes.BuildVSchema(invschema) - if err != nil { - t.Fatal(err) - } + vs := vindexes.BuildVSchema(invschema) ks := vs.Keyspaces["sharded"] ins := NewInsert( @@ -947,7 +926,7 @@ func TestInsertShardedIgnoreOwned(t *testing.T) { ksid0, } - _, err = ins.Execute(vc, map[string]*querypb.BindVariable{}, false) + _, err := ins.Execute(vc, map[string]*querypb.BindVariable{}, false) if err != nil { t.Fatal(err) } @@ -1020,10 +999,7 @@ func TestInsertShardedIgnoreOwnedWithNull(t *testing.T) { }, }, } - vs, err := vindexes.BuildVSchema(invschema) - if err != nil { - t.Fatal(err) - } + vs := vindexes.BuildVSchema(invschema) ks := vs.Keyspaces["sharded"] ins := NewInsert( @@ -1067,7 +1043,7 @@ func TestInsertShardedIgnoreOwnedWithNull(t *testing.T) { ksid0, } - _, err = ins.Execute(vc, map[string]*querypb.BindVariable{}, false) + _, err := ins.Execute(vc, map[string]*querypb.BindVariable{}, false) if err != nil { t.Fatal(err) } @@ -1122,10 +1098,7 @@ func TestInsertShardedUnownedVerify(t *testing.T) { }, }, } - vs, err := vindexes.BuildVSchema(invschema) - if err != nil { - t.Fatal(err) - } + vs := vindexes.BuildVSchema(invschema) ks := vs.Keyspaces["sharded"] ins := NewInsert( @@ -1202,7 +1175,7 @@ func TestInsertShardedUnownedVerify(t *testing.T) { nonemptyResult, nonemptyResult, } - _, err = ins.Execute(vc, map[string]*querypb.BindVariable{}, false) + _, err := ins.Execute(vc, map[string]*querypb.BindVariable{}, false) if err != nil { t.Fatal(err) } @@ -1264,10 +1237,7 @@ func TestInsertShardedIgnoreUnownedVerify(t *testing.T) { }, }, } - vs, err := vindexes.BuildVSchema(invschema) - if err != nil { - t.Fatal(err) - } + vs := vindexes.BuildVSchema(invschema) ks := vs.Keyspaces["sharded"] ins := NewInsert( @@ -1321,7 +1291,7 @@ func TestInsertShardedIgnoreUnownedVerify(t *testing.T) { {}, nonemptyResult, } - _, err = ins.Execute(vc, map[string]*querypb.BindVariable{}, false) + _, err := ins.Execute(vc, map[string]*querypb.BindVariable{}, false) if err != nil { t.Fatal(err) } @@ -1376,10 +1346,7 @@ func TestInsertShardedIgnoreUnownedVerifyFail(t *testing.T) { }, }, } - vs, err := vindexes.BuildVSchema(invschema) - if err != nil { - t.Fatal(err) - } + vs := vindexes.BuildVSchema(invschema) ks := vs.Keyspaces["sharded"] ins := NewInsert( @@ -1410,7 +1377,7 @@ func TestInsertShardedIgnoreUnownedVerifyFail(t *testing.T) { vc := newDMLTestVCursor("-20", "20-") - _, err = ins.Execute(vc, map[string]*querypb.BindVariable{}, false) + _, err := ins.Execute(vc, map[string]*querypb.BindVariable{}, false) require.EqualError(t, err, `values [[INT64(2)]] for column [c3] does not map to keyspace ids`) } @@ -1457,10 +1424,7 @@ func TestInsertShardedUnownedReverseMap(t *testing.T) { }, }, } - vs, err := vindexes.BuildVSchema(invschema) - if err != nil { - t.Fatal(err) - } + vs := vindexes.BuildVSchema(invschema) ks := vs.Keyspaces["sharded"] ins := NewInsert( @@ -1533,7 +1497,7 @@ func TestInsertShardedUnownedReverseMap(t *testing.T) { nonemptyResult, } - _, err = ins.Execute(vc, map[string]*querypb.BindVariable{}, false) + _, err := ins.Execute(vc, map[string]*querypb.BindVariable{}, false) if err != nil { t.Fatal(err) } @@ -1586,10 +1550,7 @@ func TestInsertShardedUnownedReverseMapFail(t *testing.T) { }, }, } - vs, err := vindexes.BuildVSchema(invschema) - if err != nil { - t.Fatal(err) - } + vs := vindexes.BuildVSchema(invschema) ks := vs.Keyspaces["sharded"] ins := NewInsert( @@ -1620,6 +1581,6 @@ func TestInsertShardedUnownedReverseMapFail(t *testing.T) { vc := newDMLTestVCursor("-20", "20-") - _, err = ins.Execute(vc, map[string]*querypb.BindVariable{}, false) + _, err := ins.Execute(vc, map[string]*querypb.BindVariable{}, false) require.EqualError(t, err, `value must be supplied for column [c3]`) } diff --git a/go/vt/vtgate/engine/update_test.go b/go/vt/vtgate/engine/update_test.go index fbeb63265c..979629014c 100644 --- a/go/vt/vtgate/engine/update_test.go +++ b/go/vt/vtgate/engine/update_test.go @@ -652,10 +652,7 @@ func buildTestVSchema() *vindexes.VSchema { }, }, } - vs, err := vindexes.BuildVSchema(invschema) - if err != nil { - panic(err) - } + vs := vindexes.BuildVSchema(invschema) return vs } diff --git a/go/vt/vtgate/planbuilder/plan_test.go b/go/vt/vtgate/planbuilder/plan_test.go index 9c35d472c4..073a02846b 100644 --- a/go/vt/vtgate/planbuilder/plan_test.go +++ b/go/vt/vtgate/planbuilder/plan_test.go @@ -290,7 +290,7 @@ func loadSchema(t testing.TB, filename string) *vindexes.VSchema { if err != nil { t.Fatal(err) } - vschema, err := vindexes.BuildVSchema(formal) + vschema := vindexes.BuildVSchema(formal) if err != nil { t.Fatal(err) } diff --git a/go/vt/vtgate/vindexes/vschema.go b/go/vt/vtgate/vindexes/vschema.go index 52d5af4be5..53ed49ad87 100644 --- a/go/vt/vtgate/vindexes/vschema.go +++ b/go/vt/vtgate/vindexes/vschema.go @@ -162,7 +162,7 @@ type AutoIncrement struct { } // BuildVSchema builds a VSchema from a SrvVSchema. -func BuildVSchema(source *vschemapb.SrvVSchema) (vschema *VSchema, err error) { +func BuildVSchema(source *vschemapb.SrvVSchema) (vschema *VSchema) { vschema = &VSchema{ RoutingRules: make(map[string]*RoutingRule), uniqueTables: make(map[string]*Table), @@ -173,7 +173,7 @@ func BuildVSchema(source *vschemapb.SrvVSchema) (vschema *VSchema, err error) { resolveAutoIncrement(source, vschema) addDual(vschema) buildRoutingRule(source, vschema) - return vschema, nil + return vschema } // BuildKeyspaceSchema builds the vschema portion for one keyspace. diff --git a/go/vt/vtgate/vindexes/vschema_test.go b/go/vt/vtgate/vindexes/vschema_test.go index 21ad799185..daf0708cc5 100644 --- a/go/vt/vtgate/vindexes/vschema_test.go +++ b/go/vt/vtgate/vindexes/vschema_test.go @@ -206,7 +206,7 @@ func TestUnshardedVSchema(t *testing.T) { }, }, } - got, _ := BuildVSchema(&good) + got := BuildVSchema(&good) err := got.Keyspaces["unsharded"].Error require.NoError(t, err) ks := &Keyspace{ @@ -261,7 +261,7 @@ func TestVSchemaColumns(t *testing.T) { }, }, } - got, _ := BuildVSchema(&good) + got := BuildVSchema(&good) err := got.Keyspaces["unsharded"].Error require.NoError(t, err) ks := &Keyspace{ @@ -326,8 +326,7 @@ func TestVSchemaColumnListAuthoritative(t *testing.T) { }, }, } - got, err := BuildVSchema(&good) - require.NoError(t, err) + got := BuildVSchema(&good) ks := &Keyspace{ Name: "unsharded", } @@ -389,7 +388,7 @@ func TestVSchemaColumnsFail(t *testing.T) { }, }, } - got, _ := BuildVSchema(&good) + got := BuildVSchema(&good) want := "duplicate column name 'c1' for table: t1" err := got.Keyspaces["unsharded"].Error if err == nil || err.Error() != want { @@ -410,7 +409,7 @@ func TestVSchemaPinned(t *testing.T) { }, }, } - got, _ := BuildVSchema(&good) + got := BuildVSchema(&good) err := got.Keyspaces["sharded"].Error require.NoError(t, err) ks := &Keyspace{ @@ -486,7 +485,7 @@ func TestShardedVSchemaOwned(t *testing.T) { }, }, } - got, _ := BuildVSchema(&good) + got := BuildVSchema(&good) err := got.Keyspaces["sharded"].Error require.NoError(t, err) ks := &Keyspace{ @@ -614,7 +613,7 @@ func TestShardedVSchemaOwnerInfo(t *testing.T) { }, }, } - got, _ := BuildVSchema(&good) + got := BuildVSchema(&good) err := got.Keyspaces["sharded"].Error require.NoError(t, err) results := []struct { @@ -714,7 +713,7 @@ func TestVSchemaRoutingRules(t *testing.T) { }, }, } - got, _ := BuildVSchema(&input) + got := BuildVSchema(&input) ks1 := &Keyspace{ Name: "ks1", Sharded: true, @@ -987,8 +986,7 @@ func TestFindBestColVindex(t *testing.T) { }, }, } - vschema, err := BuildVSchema(testSrvVSchema) - require.NoError(t, err) + vs := BuildVSchema(testSrvVSchema) testcases := []struct { tablename string @@ -1011,7 +1009,7 @@ func TestFindBestColVindex(t *testing.T) { err: "table t2 has no vindex", }} for _, tcase := range testcases { - table, err := vschema.FindTable("", tcase.tablename) + table, err := vs.FindTable("", tcase.tablename) require.NoError(t, err) cv, err := FindBestColVindex(table) if err != nil { @@ -1162,7 +1160,7 @@ func TestShardedVSchemaMultiColumnVindex(t *testing.T) { }, }, } - got, _ := BuildVSchema(&good) + got := BuildVSchema(&good) err := got.Keyspaces["sharded"].Error require.NoError(t, err) ks := &Keyspace{ @@ -1255,7 +1253,7 @@ func TestShardedVSchemaNotOwned(t *testing.T) { }, }, } - got, _ := BuildVSchema(&good) + got := BuildVSchema(&good) err := got.Keyspaces["sharded"].Error require.NoError(t, err) ks := &Keyspace{ @@ -1347,7 +1345,7 @@ func TestBuildVSchemaVindexNotFoundFail(t *testing.T) { }, }, } - got, _ := BuildVSchema(&bad) + got := BuildVSchema(&bad) err := got.Keyspaces["sharded"].Error want := `vindexType "noexist" not found` if err == nil || err.Error() != want { @@ -1371,7 +1369,7 @@ func TestBuildVSchemaNoColumnVindexFail(t *testing.T) { }, }, } - got, _ := BuildVSchema(&bad) + got := BuildVSchema(&bad) err := got.Keyspaces["sharded"].Error want := "missing primary col vindex for table: t1" if err == nil || err.Error() != want { @@ -1404,7 +1402,7 @@ func TestBuildVSchemaDupSeq(t *testing.T) { ksb := &Keyspace{ Name: "ksb", } - got, _ := BuildVSchema(&good) + got := BuildVSchema(&good) t1a := &Table{ Name: sqlparser.NewTableIdent("t1"), Keyspace: ksa, @@ -1473,7 +1471,7 @@ func TestBuildVSchemaDupTable(t *testing.T) { }, }, } - got, _ := BuildVSchema(&good) + got := BuildVSchema(&good) ksa := &Keyspace{ Name: "ksa", } @@ -1574,7 +1572,7 @@ func TestBuildVSchemaDupVindex(t *testing.T) { }, }, } - got, _ := BuildVSchema(&good) + got := BuildVSchema(&good) err := got.Keyspaces["ksa"].Error err1 := got.Keyspaces["ksb"].Error require.NoError(t, err) @@ -1694,7 +1692,7 @@ func TestBuildVSchemaNoindexFail(t *testing.T) { }, }, } - got, _ := BuildVSchema(&bad) + got := BuildVSchema(&bad) err := got.Keyspaces["sharded"].Error want := "vindex notexist not found for table t1" if err == nil || err.Error() != want { @@ -1726,7 +1724,7 @@ func TestBuildVSchemaColumnAndColumnsFail(t *testing.T) { }, }, } - got, _ := BuildVSchema(&bad) + got := BuildVSchema(&bad) err := got.Keyspaces["sharded"].Error want := `can't use column and columns at the same time in vindex (stfu) and table (t1)` if err == nil || err.Error() != want { @@ -1756,7 +1754,7 @@ func TestBuildVSchemaNoColumnsFail(t *testing.T) { }, }, } - got, _ := BuildVSchema(&bad) + got := BuildVSchema(&bad) err := got.Keyspaces["sharded"].Error want := `must specify at least one column for vindex (stfu) and table (t1)` if err == nil || err.Error() != want { @@ -1787,7 +1785,7 @@ func TestBuildVSchemaNotUniqueFail(t *testing.T) { }, }, } - got, _ := BuildVSchema(&bad) + got := BuildVSchema(&bad) err := got.Keyspaces["sharded"].Error want := "primary vindex stln is not Unique for table t1" if err == nil || err.Error() != want { @@ -1819,7 +1817,7 @@ func TestBuildVSchemaPrimaryCannotBeOwned(t *testing.T) { }, }, } - got, _ := BuildVSchema(&bad) + got := BuildVSchema(&bad) err := got.Keyspaces["sharded"].Error want := "primary vindex stlu cannot be owned for table t1" if err == nil || err.Error() != want { @@ -1876,7 +1874,7 @@ func TestSequence(t *testing.T) { }, }, } - got, _ := BuildVSchema(&good) + got := BuildVSchema(&good) err := got.Keyspaces["sharded"].Error require.NoError(t, err) err1 := got.Keyspaces["unsharded"].Error @@ -2035,7 +2033,7 @@ func TestBadSequence(t *testing.T) { }, }, } - got, _ := BuildVSchema(&bad) + got := BuildVSchema(&bad) err := got.Keyspaces["sharded"].Error want := "cannot resolve sequence invalid_seq: table invalid_seq not found" if err == nil || err.Error() != want { @@ -2083,7 +2081,7 @@ func TestBadSequenceName(t *testing.T) { }, }, } - got, _ := BuildVSchema(&bad) + got := BuildVSchema(&bad) err := got.Keyspaces["sharded"].Error want := "invalid table name: a.b.seq" if err == nil || !strings.Contains(err.Error(), want) { @@ -2107,7 +2105,7 @@ func TestBadShardedSequence(t *testing.T) { }, }, } - got, _ := BuildVSchema(&bad) + got := BuildVSchema(&bad) err := got.Keyspaces["sharded"].Error want := "sequence table has to be in an unsharded keyspace or must be pinned: t1" if err == nil || err.Error() != want { @@ -2155,7 +2153,7 @@ func TestFindTable(t *testing.T) { }, }, } - vschema, _ := BuildVSchema(&input) + vschema := BuildVSchema(&input) _, err := vschema.FindTable("", "t1") require.EqualError(t, err, "ambiguous table reference: t1") @@ -2253,7 +2251,7 @@ func TestFindTableOrVindex(t *testing.T) { }, }, } - vschema, _ := BuildVSchema(&input) + vschema := BuildVSchema(&input) ta := vschema.Keyspaces["ksa"].Tables["ta"] t1 := vschema.Keyspaces["ksb"].Tables["t1"] @@ -2596,7 +2594,7 @@ func TestFindSingleKeyspace(t *testing.T) { }, }, } - vschema, _ := BuildVSchema(&input) + vschema := BuildVSchema(&input) none := &Table{ Name: sqlparser.NewTableIdent("none"), Keyspace: &Keyspace{ @@ -2637,7 +2635,7 @@ func TestFindSingleKeyspace(t *testing.T) { }, }, } - vschema, _ = BuildVSchema(&input) + vschema = BuildVSchema(&input) _, err := vschema.FindTable("", "none") wantErr := "table none not found" if err == nil || err.Error() != wantErr { diff --git a/go/vt/vtgate/vschema_manager.go b/go/vt/vtgate/vschema_manager.go index 2db5dc691b..cc10197840 100644 --- a/go/vt/vtgate/vschema_manager.go +++ b/go/vt/vtgate/vschema_manager.go @@ -18,7 +18,6 @@ package vtgate import ( "context" - "fmt" "sync" "vitess.io/vitess/go/vt/sqlparser" @@ -120,10 +119,10 @@ func (vm *VSchemaManager) VSchemaUpdate(v *vschemapb.SrvVSchema, err error) { if v == nil { // We encountered an error, build an empty vschema. if vm.currentVschema == nil { - vschema, _ = vindexes.BuildVSchema(&vschemapb.SrvVSchema{}) + vschema = vindexes.BuildVSchema(&vschemapb.SrvVSchema{}) } } else { - vschema, err = vm.buildAndEnhanceVSchema(v) + vschema = vm.buildAndEnhanceVSchema(v) vm.currentVschema = vschema } @@ -155,39 +154,27 @@ func (vm *VSchemaManager) Rebuild() { v := vm.currentSrvVschema vm.mu.Unlock() - var vschema *vindexes.VSchema - var err error - if v == nil { - v = &vschemapb.SrvVSchema{} - } - - vschema, err = vm.buildAndEnhanceVSchema(v) - if err != nil { - log.Error("failed to reload vschema after schema change") return } + vschema := vm.buildAndEnhanceVSchema(v) + vm.mu.Lock() + vm.currentVschema = vschema + vm.mu.Unlock() + if vm.subscriber != nil { - vm.subscriber(vschema, vSchemaStats(err, vschema)) + vm.subscriber(vschema, vSchemaStats(nil, vschema)) } } // buildAndEnhanceVSchema builds a new VSchema and uses information from the schema tracker to update it -func (vm *VSchemaManager) buildAndEnhanceVSchema(v *vschemapb.SrvVSchema) (*vindexes.VSchema, error) { - vschema, err := vindexes.BuildVSchema(v) - if err == nil { - if vm.schema != nil { - vm.updateFromSchema(vschema) - } - } else { - log.Warningf("Error creating VSchema for cell %v (will try again next update): %v", vm.cell, err) - err = fmt.Errorf("error creating VSchema for cell %v: %v", vm.cell, err) - if vschemaCounters != nil { - vschemaCounters.Add("Parsing", 1) - } +func (vm *VSchemaManager) buildAndEnhanceVSchema(v *vschemapb.SrvVSchema) *vindexes.VSchema { + vschema := vindexes.BuildVSchema(v) + if vm.schema != nil { + vm.updateFromSchema(vschema) } - return vschema, err + return vschema } func (vm *VSchemaManager) updateFromSchema(vschema *vindexes.VSchema) { diff --git a/go/vt/vtgate/vschema_manager_test.go b/go/vt/vtgate/vschema_manager_test.go index 976889907e..7e166e7786 100644 --- a/go/vt/vtgate/vschema_manager_test.go +++ b/go/vt/vtgate/vschema_manager_test.go @@ -3,8 +3,6 @@ package vtgate import ( "testing" - "github.com/stretchr/testify/require" - "vitess.io/vitess/go/test/utils" querypb "vitess.io/vitess/go/vt/proto/query" "vitess.io/vitess/go/vt/sqlparser" @@ -13,13 +11,13 @@ import ( "vitess.io/vitess/go/vt/vtgate/vindexes" ) -func TestWatchSrvVSchema(t *testing.T) { - cols := []vindexes.Column{{ +func TestVSchemaUpdate(t *testing.T) { + cols1 := []vindexes.Column{{ Name: sqlparser.NewColIdent("id"), Type: querypb.Type_INT64, }} cols2 := []vindexes.Column{{ - Name: sqlparser.NewColIdent("id"), + Name: sqlparser.NewColIdent("uid"), Type: querypb.Type_INT64, }, { Name: sqlparser.NewColIdent("name"), @@ -27,75 +25,76 @@ func TestWatchSrvVSchema(t *testing.T) { }} ks := &vindexes.Keyspace{Name: "ks"} dual := &vindexes.Table{Type: vindexes.TypeReference, Name: sqlparser.NewTableIdent("dual"), Keyspace: ks} + tblNoCol := &vindexes.Table{Name: sqlparser.NewTableIdent("tbl"), Keyspace: ks, ColumnListAuthoritative: true} + tblCol1 := &vindexes.Table{Name: sqlparser.NewTableIdent("tbl"), Keyspace: ks, Columns: cols1, ColumnListAuthoritative: true} + tblCol2 := &vindexes.Table{Name: sqlparser.NewTableIdent("tbl"), Keyspace: ks, Columns: cols2, ColumnListAuthoritative: true} + tblCol2NA := &vindexes.Table{Name: sqlparser.NewTableIdent("tbl"), Keyspace: ks, Columns: cols2} + tcases := []struct { - name string - srvVschema *vschemapb.SrvVSchema - schema map[string][]vindexes.Column - expected map[string]*vindexes.Table + name string + srvVschema *vschemapb.SrvVSchema + currentVSchema *vindexes.VSchema + schema map[string][]vindexes.Column + expected *vindexes.VSchema }{{ - name: "Single table known by mysql schema and not by vschema", - srvVschema: &vschemapb.SrvVSchema{Keyspaces: map[string]*vschemapb.Keyspace{"ks": {}}}, - schema: map[string][]vindexes.Column{"tbl": cols}, - expected: map[string]*vindexes.Table{ - "dual": dual, + name: "0 Schematracking- 1 srvVSchema", + srvVschema: makeTestSrvVSchema("ks", false, map[string]*vschemapb.Table{ "tbl": { - Name: sqlparser.NewTableIdent("tbl"), - Keyspace: ks, - Columns: cols, - ColumnListAuthoritative: true, + Columns: []*vschemapb.Column{{Name: "uid", Type: querypb.Type_INT64}, {Name: "name", Type: querypb.Type_VARCHAR}}, + ColumnListAuthoritative: false, }, - }, + }), + expected: makeTestVSchema("ks", false, map[string]*vindexes.Table{"dual": dual, "tbl": tblCol2NA}), }, { - name: "Single table known by both - vschema is not authoritative", - srvVschema: &vschemapb.SrvVSchema{Keyspaces: map[string]*vschemapb.Keyspace{"ks": { - Tables: map[string]*vschemapb.Table{ - "tbl": {}, // we know of it, but nothing else - }, - }}}, - schema: map[string][]vindexes.Column{"tbl": cols}, - expected: map[string]*vindexes.Table{ - "dual": dual, - "tbl": { - Name: sqlparser.NewTableIdent("tbl"), - Keyspace: ks, - Columns: cols, - ColumnListAuthoritative: true, - }, - }, + name: "1 Schematracking- 0 srvVSchema", + srvVschema: makeTestSrvVSchema("ks", false, nil), + schema: map[string][]vindexes.Column{"tbl": cols1}, + expected: makeTestVSchema("ks", false, map[string]*vindexes.Table{"dual": dual, "tbl": tblCol1}), }, { - name: "Single table known by both - vschema is authoritative", - srvVschema: &vschemapb.SrvVSchema{Keyspaces: map[string]*vschemapb.Keyspace{"ks": { - Tables: map[string]*vschemapb.Table{ - "tbl": { - Columns: []*vschemapb.Column{ - {Name: "id", Type: querypb.Type_INT64}, - {Name: "name", Type: querypb.Type_VARCHAR}, - }, - ColumnListAuthoritative: true}, - }, - }}}, - schema: map[string][]vindexes.Column{"tbl": cols}, - expected: map[string]*vindexes.Table{ - "dual": dual, - "tbl": { - Name: sqlparser.NewTableIdent("tbl"), - Keyspace: ks, - Columns: cols2, - ColumnListAuthoritative: true, - }, - }, + name: "1 Schematracking - 1 srvVSchema (no columns) not authoritative", + srvVschema: makeTestSrvVSchema("ks", false, map[string]*vschemapb.Table{"tbl": {}}), + schema: map[string][]vindexes.Column{"tbl": cols1}, + // schema will override what srvSchema has. + expected: makeTestVSchema("ks", false, map[string]*vindexes.Table{"dual": dual, "tbl": tblCol1}), }, { - name: "empty srvVschema - schema change still applied", - schema: map[string][]vindexes.Column{"tbl": cols}, - expected: map[string]*vindexes.Table{ - "dual": dual, + name: "1 Schematracking - 1 srvVSchema (have columns) not authoritative", + srvVschema: makeTestSrvVSchema("ks", false, map[string]*vschemapb.Table{ "tbl": { - Name: sqlparser.NewTableIdent("tbl"), - Keyspace: ks, - Columns: cols2, + Columns: []*vschemapb.Column{{Name: "uid", Type: querypb.Type_INT64}, {Name: "name", Type: querypb.Type_VARCHAR}}, + ColumnListAuthoritative: false, + }, + }), + schema: map[string][]vindexes.Column{"tbl": cols1}, + // schema will override what srvSchema has. + expected: makeTestVSchema("ks", false, map[string]*vindexes.Table{"dual": dual, "tbl": tblCol1}), + }, { + name: "1 Schematracking - 1 srvVSchema (no columns) authoritative", + srvVschema: makeTestSrvVSchema("ks", false, map[string]*vschemapb.Table{"tbl": { + ColumnListAuthoritative: true, + }}), + schema: map[string][]vindexes.Column{"tbl": cols1}, + // schema will override what srvSchema has. + expected: makeTestVSchema("ks", false, map[string]*vindexes.Table{"dual": dual, "tbl": tblNoCol}), + }, { + name: "1 Schematracking - 1 srvVSchema (have columns) authoritative", + srvVschema: makeTestSrvVSchema("ks", false, map[string]*vschemapb.Table{ + "tbl": { + Columns: []*vschemapb.Column{{Name: "uid", Type: querypb.Type_INT64}, {Name: "name", Type: querypb.Type_VARCHAR}}, ColumnListAuthoritative: true, }, - }, + }), + schema: map[string][]vindexes.Column{"tbl": cols1}, + // schema tracker will be ignored for authoritative tables. + expected: makeTestVSchema("ks", false, map[string]*vindexes.Table{"dual": dual, "tbl": tblCol2}), + }, { + name: "srvVschema received as nil", + schema: map[string][]vindexes.Column{"tbl": cols1}, + expected: makeTestEmptyVSchema(), + }, { + name: "srvVschema received as nil - have existing vschema", + currentVSchema: &vindexes.VSchema{}, + schema: map[string][]vindexes.Column{"tbl": cols1}, + expected: &vindexes.VSchema{}, }} vm := &VSchemaManager{} @@ -104,38 +103,152 @@ func TestWatchSrvVSchema(t *testing.T) { vs = vschema } for _, tcase := range tcases { - t.Run("VSchemaUpdate - "+tcase.name, func(t *testing.T) { + t.Run(tcase.name, func(t *testing.T) { vs = nil vm.schema = &fakeSchema{t: tcase.schema} + vm.currentSrvVschema = nil + vm.currentVschema = tcase.currentVSchema vm.VSchemaUpdate(tcase.srvVschema, nil) - require.NotNil(t, vs) - ks := vs.Keyspaces["ks"] + utils.MustMatchFn(".uniqueTables", ".uniqueVindexes")(t, tcase.expected, vs) if tcase.srvVschema != nil { - require.NotNil(t, ks, "keyspace was not found") - utils.MustMatch(t, tcase.expected, ks.Tables) - } else { - require.Nil(t, ks, "keyspace found") + utils.MustMatch(t, vs, vm.currentVschema, "currentVschema should have same reference as Vschema") } }) - t.Run("Schema updated - "+tcase.name, func(t *testing.T) { + } +} + +func TestRebuildVSchema(t *testing.T) { + cols1 := []vindexes.Column{{ + Name: sqlparser.NewColIdent("id"), + Type: querypb.Type_INT64, + }} + cols2 := []vindexes.Column{{ + Name: sqlparser.NewColIdent("uid"), + Type: querypb.Type_INT64, + }, { + Name: sqlparser.NewColIdent("name"), + Type: querypb.Type_VARCHAR, + }} + ks := &vindexes.Keyspace{Name: "ks"} + dual := &vindexes.Table{Type: vindexes.TypeReference, Name: sqlparser.NewTableIdent("dual"), Keyspace: ks} + tblNoCol := &vindexes.Table{Name: sqlparser.NewTableIdent("tbl"), Keyspace: ks, ColumnListAuthoritative: true} + tblCol1 := &vindexes.Table{Name: sqlparser.NewTableIdent("tbl"), Keyspace: ks, Columns: cols1, ColumnListAuthoritative: true} + tblCol2 := &vindexes.Table{Name: sqlparser.NewTableIdent("tbl"), Keyspace: ks, Columns: cols2, ColumnListAuthoritative: true} + tblCol2NA := &vindexes.Table{Name: sqlparser.NewTableIdent("tbl"), Keyspace: ks, Columns: cols2} + + tcases := []struct { + name string + srvVschema *vschemapb.SrvVSchema + schema map[string][]vindexes.Column + expected *vindexes.VSchema + }{{ + name: "0 Schematracking- 1 srvVSchema", + srvVschema: makeTestSrvVSchema("ks", false, map[string]*vschemapb.Table{ + "tbl": { + Columns: []*vschemapb.Column{{Name: "uid", Type: querypb.Type_INT64}, {Name: "name", Type: querypb.Type_VARCHAR}}, + ColumnListAuthoritative: false, + }, + }), + expected: makeTestVSchema("ks", false, map[string]*vindexes.Table{"dual": dual, "tbl": tblCol2NA}), + }, { + name: "1 Schematracking- 0 srvVSchema", + srvVschema: makeTestSrvVSchema("ks", false, nil), + schema: map[string][]vindexes.Column{"tbl": cols1}, + expected: makeTestVSchema("ks", false, map[string]*vindexes.Table{"dual": dual, "tbl": tblCol1}), + }, { + name: "1 Schematracking - 1 srvVSchema (no columns) not authoritative", + srvVschema: makeTestSrvVSchema("ks", false, map[string]*vschemapb.Table{"tbl": {}}), + schema: map[string][]vindexes.Column{"tbl": cols1}, + // schema will override what srvSchema has. + expected: makeTestVSchema("ks", false, map[string]*vindexes.Table{"dual": dual, "tbl": tblCol1}), + }, { + name: "1 Schematracking - 1 srvVSchema (have columns) not authoritative", + srvVschema: makeTestSrvVSchema("ks", false, map[string]*vschemapb.Table{ + "tbl": { + Columns: []*vschemapb.Column{{Name: "uid", Type: querypb.Type_INT64}, {Name: "name", Type: querypb.Type_VARCHAR}}, + ColumnListAuthoritative: false, + }, + }), + schema: map[string][]vindexes.Column{"tbl": cols1}, + // schema will override what srvSchema has. + expected: makeTestVSchema("ks", false, map[string]*vindexes.Table{"dual": dual, "tbl": tblCol1}), + }, { + name: "1 Schematracking - 1 srvVSchema (no columns) authoritative", + srvVschema: makeTestSrvVSchema("ks", false, map[string]*vschemapb.Table{"tbl": { + ColumnListAuthoritative: true, + }}), + schema: map[string][]vindexes.Column{"tbl": cols1}, + // schema will override what srvSchema has. + expected: makeTestVSchema("ks", false, map[string]*vindexes.Table{"dual": dual, "tbl": tblNoCol}), + }, { + name: "1 Schematracking - 1 srvVSchema (have columns) authoritative", + srvVschema: makeTestSrvVSchema("ks", false, map[string]*vschemapb.Table{ + "tbl": { + Columns: []*vschemapb.Column{{Name: "uid", Type: querypb.Type_INT64}, {Name: "name", Type: querypb.Type_VARCHAR}}, + ColumnListAuthoritative: true, + }, + }), + schema: map[string][]vindexes.Column{"tbl": cols1}, + // schema tracker will be ignored for authoritative tables. + expected: makeTestVSchema("ks", false, map[string]*vindexes.Table{"dual": dual, "tbl": tblCol2}), + }, { + name: "srvVschema received as nil", + schema: map[string][]vindexes.Column{"tbl": cols1}, + }} + + vm := &VSchemaManager{} + var vs *vindexes.VSchema + vm.subscriber = func(vschema *vindexes.VSchema, _ *VSchemaStats) { + vs = vschema + } + for _, tcase := range tcases { + t.Run(tcase.name, func(t *testing.T) { vs = nil vm.schema = &fakeSchema{t: tcase.schema} vm.currentSrvVschema = tcase.srvVschema + vm.currentVschema = nil vm.Rebuild() - require.NotNil(t, vs) - ks := vs.Keyspaces["ks"] - if tcase.srvVschema != nil { - require.NotNil(t, ks, "keyspace was not found") - utils.MustMatch(t, tcase.expected, ks.Tables) - } else { - require.Nil(t, ks, "keyspace found") + utils.MustMatchFn(".uniqueTables", ".uniqueVindexes")(t, tcase.expected, vs) + if vs != nil { + utils.MustMatch(t, vs, vm.currentVschema, "currentVschema should have same reference as Vschema") } }) } } +func makeTestVSchema(ks string, sharded bool, tbls map[string]*vindexes.Table) *vindexes.VSchema { + kSchema := &vindexes.KeyspaceSchema{ + Keyspace: &vindexes.Keyspace{ + Name: ks, + Sharded: sharded, + }, + Tables: tbls, + Vindexes: map[string]vindexes.Vindex{}, + } + vs := makeTestEmptyVSchema() + vs.Keyspaces[ks] = kSchema + return vs +} + +func makeTestEmptyVSchema() *vindexes.VSchema { + return &vindexes.VSchema{ + RoutingRules: map[string]*vindexes.RoutingRule{}, + Keyspaces: map[string]*vindexes.KeyspaceSchema{}, + } +} + +func makeTestSrvVSchema(ks string, sharded bool, tbls map[string]*vschemapb.Table) *vschemapb.SrvVSchema { + kSchema := &vschemapb.Keyspace{ + Sharded: sharded, + Tables: tbls, + } + return &vschemapb.SrvVSchema{ + Keyspaces: map[string]*vschemapb.Keyspace{ks: kSchema}, + } +} + type fakeSchema struct { t map[string][]vindexes.Column } diff --git a/go/vt/vttablet/tabletserver/vstreamer/engine.go b/go/vt/vttablet/tabletserver/vstreamer/engine.go index c8a5b60a59..6c604355e1 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/engine.go +++ b/go/vt/vttablet/tabletserver/vstreamer/engine.go @@ -346,7 +346,7 @@ func (vse *Engine) setWatch() { } var vschema *vindexes.VSchema if v != nil { - vschema, err = vindexes.BuildVSchema(v) + vschema = vindexes.BuildVSchema(v) if err != nil { log.Errorf("Error building vschema: %v", err) vse.vschemaErrors.Add(1) diff --git a/go/vt/vttablet/tabletserver/vstreamer/fuzz.go b/go/vt/vttablet/tabletserver/vstreamer/fuzz.go index a47011c60d..a36100e331 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/fuzz.go +++ b/go/vt/vttablet/tabletserver/vstreamer/fuzz.go @@ -38,7 +38,7 @@ func Fuzz(data []byte) int { "ks": &kspb, }, } - vschema, err := vindexes.BuildVSchema(srvVSchema) + vschema := vindexes.BuildVSchema(srvVSchema) if err != nil { return -1 } diff --git a/go/vt/vttablet/tabletserver/vstreamer/local_vschema_test.go b/go/vt/vttablet/tabletserver/vstreamer/local_vschema_test.go index 31d1780f4f..f514298e84 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/local_vschema_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/local_vschema_test.go @@ -21,7 +21,6 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" vschemapb "vitess.io/vitess/go/vt/proto/vschema" "vitess.io/vitess/go/vt/vtgate/vindexes" @@ -87,8 +86,7 @@ func TestFindColVindex(t *testing.T) { }, }, } - vschema, err := vindexes.BuildVSchema(testSrvVSchema) - require.NoError(t, err) + vschema := vindexes.BuildVSchema(testSrvVSchema) testcases := []struct { keyspace string @@ -151,8 +149,7 @@ func TestFindOrCreateVindex(t *testing.T) { }, }, } - vschema, err := vindexes.BuildVSchema(testSrvVSchema) - require.NoError(t, err) + vschema := vindexes.BuildVSchema(testSrvVSchema) lvs := &localVSchema{ keyspace: "ks1", @@ -207,8 +204,7 @@ func TestFindTable(t *testing.T) { }, }, } - vschema, err := vindexes.BuildVSchema(testSrvVSchema) - require.NoError(t, err) + vschema := vindexes.BuildVSchema(testSrvVSchema) testcases := []struct { keyspace string diff --git a/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go b/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go index 35c1e5d41f..de45787fca 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go @@ -83,10 +83,7 @@ func init() { "ks": &kspb, }, } - vschema, err := vindexes.BuildVSchema(srvVSchema) - if err != nil { - panic(err) - } + vschema := vindexes.BuildVSchema(srvVSchema) testLocalVSchema = &localVSchema{ keyspace: "ks", vschema: vschema, From 0b76c1d794eaf06c8fd0b1c8843e130d8a637db0 Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Thu, 3 Jun 2021 15:48:53 +0200 Subject: [PATCH 15/64] New E2E tests for new tables in unsharded/sharded keyspaces Signed-off-by: Florent Poinsard --- go/test/endtoend/vtgate/main_test.go | 2 +- go/test/endtoend/vtgate/misc_test.go | 20 ++++ .../schematracker/schematracker_test.go | 107 ++++++++++++++++++ test/config.json | 9 ++ 4 files changed, 137 insertions(+), 1 deletion(-) create mode 100644 go/test/endtoend/vtgate/schematracker/schematracker_test.go diff --git a/go/test/endtoend/vtgate/main_test.go b/go/test/endtoend/vtgate/main_test.go index 7fda9e8fbe..ecce216b54 100644 --- a/go/test/endtoend/vtgate/main_test.go +++ b/go/test/endtoend/vtgate/main_test.go @@ -418,7 +418,7 @@ func TestMain(m *testing.M) { VSchema: VSchema, } clusterInstance.VtGateExtraArgs = []string{"-schema_change_signal"} - clusterInstance.VtTabletExtraArgs = []string{"-queryserver-config-schema-change-signal"} + clusterInstance.VtTabletExtraArgs = []string{"-queryserver-config-schema-change-signal", "-queryserver-config-schema-change-signal-interval", "1"} err = clusterInstance.StartKeyspace(*keyspace, []string{"-80", "80-"}, 1, true) if err != nil { return 1 diff --git a/go/test/endtoend/vtgate/misc_test.go b/go/test/endtoend/vtgate/misc_test.go index 77de7f4076..888bf7346f 100644 --- a/go/test/endtoend/vtgate/misc_test.go +++ b/go/test/endtoend/vtgate/misc_test.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "testing" + "time" "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/assert" @@ -674,6 +675,25 @@ func TestVSchemaTrackerInit(t *testing.T) { assert.Equal(t, want, got) } +func TestVSchemaTrackedForNewTables(t *testing.T) { + ctx := context.Background() + conn, err := mysql.Connect(ctx, &vtParams) + require.NoError(t, err) + defer conn.Close() + + // create a new table which is not part of the VSchema + exec(t, conn, `create table new_table_tracked(id bigint, name varchar(100), primary key(id)) Engine=InnoDB`) + + // wait for vttablet's schema reload interval to pass + time.Sleep(2 * time.Second) + + // check if the new table is part of the schema + qr := exec(t, conn, "SHOW VSCHEMA TABLES") + got := fmt.Sprintf("%v", qr.Rows) + want := `[[VARCHAR("aggr_test")] [VARCHAR("dual")] [VARCHAR("new_table_tracked")] [VARCHAR("t1")] [VARCHAR("t1_id2_idx")] [VARCHAR("t2")] [VARCHAR("t2_id4_idx")] [VARCHAR("t3")] [VARCHAR("t3_id7_idx")] [VARCHAR("t4")] [VARCHAR("t4_id2_idx")] [VARCHAR("t5_null_vindex")] [VARCHAR("t6")] [VARCHAR("t6_id2_idx")] [VARCHAR("t7_fk")] [VARCHAR("t7_xxhash")] [VARCHAR("t7_xxhash_idx")] [VARCHAR("t8")] [VARCHAR("vstream_test")]]` + assert.Equal(t, want, got) +} + func assertMatches(t *testing.T, conn *mysql.Conn, query, expected string) { t.Helper() qr := exec(t, conn, query) diff --git a/go/test/endtoend/vtgate/schematracker/schematracker_test.go b/go/test/endtoend/vtgate/schematracker/schematracker_test.go new file mode 100644 index 0000000000..f114c76b8c --- /dev/null +++ b/go/test/endtoend/vtgate/schematracker/schematracker_test.go @@ -0,0 +1,107 @@ +/* +Copyright 2021 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package schematracker + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/test/endtoend/cluster" +) + +var ( + hostname = "localhost" + keyspaceName = "ks" + cell = "zone1" + sqlSchema = ` + create table main ( + id bigint, + val varchar(128), + primary key(id) + ) Engine=InnoDB; +` +) + +func TestNewUnshardedTable(t *testing.T) { + defer cluster.PanicHandler(t) + var err error + + // initialize our cluster + clusterInstance := cluster.NewCluster(cell, hostname) + defer clusterInstance.Teardown() + + // Start topo server + err = clusterInstance.StartTopo() + require.NoError(t, err) + + // create keyspace + keyspace := &cluster.Keyspace{ + Name: keyspaceName, + SchemaSQL: sqlSchema, + } + + // enabling and setting the schema reload time to one second + clusterInstance.VtTabletExtraArgs = []string{"-queryserver-config-schema-change-signal", "-queryserver-config-schema-change-signal-interval", "1"} + err = clusterInstance.StartUnshardedKeyspace(*keyspace, 1, false) + require.NoError(t, err) + + // Start vtgate with the schema_change_signal flag + clusterInstance.VtGateExtraArgs = []string{"-schema_change_signal"} + err = clusterInstance.StartVtgate() + require.NoError(t, err) + + // create a sql connection + ctx := context.Background() + conn, err := mysql.Connect(ctx, &mysql.ConnParams{ + Host: clusterInstance.Hostname, + Port: clusterInstance.VtgateMySQLPort, + }) + require.NoError(t, err) + defer conn.Close() + + // ensuring our initial table "main" is in the schema + qr := exec(t, conn, "SHOW VSCHEMA TABLES") + got := fmt.Sprintf("%v", qr.Rows) + want := `[[VARCHAR("dual")] [VARCHAR("main")]]` + require.Equal(t, want, got) + + // create a new table which is not part of the VSchema + exec(t, conn, `create table new_table_tracked(id bigint, name varchar(100), primary key(id)) Engine=InnoDB`) + + // waiting for the vttablet's schema_reload interval to kick in + time.Sleep(2 * time.Second) + + // ensuring our new table is in the schema + qr = exec(t, conn, "SHOW VSCHEMA TABLES") + got = fmt.Sprintf("%v", qr.Rows) + want = `[[VARCHAR("dual")] [VARCHAR("main")] [VARCHAR("new_table_tracked")]]` + assert.Equal(t, want, got) +} + +func exec(t *testing.T, conn *mysql.Conn, query string) *sqltypes.Result { + t.Helper() + qr, err := conn.ExecuteFetch(query, 1000, true) + require.NoError(t, err, "for query: "+query) + return qr +} diff --git a/test/config.json b/test/config.json index 61fc166067..088d815ea7 100644 --- a/test/config.json +++ b/test/config.json @@ -606,6 +606,15 @@ "RetryMax": 0, "Tags": [] }, + "vtgate_schematracker": { + "File": "unused.go", + "Args": ["vitess.io/vitess/go/test/endtoend/vtgate/schematracker"], + "Command": [], + "Manual": false, + "Shard": "17", + "RetryMax": 0, + "Tags": [] + }, "vtgate_sequence": { "File": "unused.go", "Args": ["vitess.io/vitess/go/test/endtoend/vtgate/sequence"], From 97ce0972f1a8cd775e3462ae7a1ad894ecda0f86 Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Thu, 3 Jun 2021 17:25:17 +0200 Subject: [PATCH 16/64] Improved vschema tracker misc end to end tests Signed-off-by: Florent Poinsard --- go/test/endtoend/vtgate/main_test.go | 14 ++++++++ go/test/endtoend/vtgate/misc_test.go | 33 +++++++++++++++---- .../schematracker/schematracker_test.go | 4 +-- 3 files changed, 43 insertions(+), 8 deletions(-) diff --git a/go/test/endtoend/vtgate/main_test.go b/go/test/endtoend/vtgate/main_test.go index ecce216b54..6b2ff4b4a0 100644 --- a/go/test/endtoend/vtgate/main_test.go +++ b/go/test/endtoend/vtgate/main_test.go @@ -142,6 +142,12 @@ create table t8( testId bigint, primary key(id8) ) Engine=InnoDB; + +create table t9( + id9 bigint, + testId bigint, + primary key(id9) +) Engine=InnoDB; ` VSchema = ` @@ -384,6 +390,14 @@ create table t8( "name": "hash" } ] + }, + "t9": { + "column_vindexes": [ + { + "column": "id9", + "name": "hash" + } + ] } } }` diff --git a/go/test/endtoend/vtgate/misc_test.go b/go/test/endtoend/vtgate/misc_test.go index 888bf7346f..32b4f3b713 100644 --- a/go/test/endtoend/vtgate/misc_test.go +++ b/go/test/endtoend/vtgate/misc_test.go @@ -671,8 +671,21 @@ func TestVSchemaTrackerInit(t *testing.T) { qr := exec(t, conn, "SHOW VSCHEMA TABLES") got := fmt.Sprintf("%v", qr.Rows) - want := `[[VARCHAR("aggr_test")] [VARCHAR("dual")] [VARCHAR("t1")] [VARCHAR("t1_id2_idx")] [VARCHAR("t2")] [VARCHAR("t2_id4_idx")] [VARCHAR("t3")] [VARCHAR("t3_id7_idx")] [VARCHAR("t4")] [VARCHAR("t4_id2_idx")] [VARCHAR("t5_null_vindex")] [VARCHAR("t6")] [VARCHAR("t6_id2_idx")] [VARCHAR("t7_fk")] [VARCHAR("t7_xxhash")] [VARCHAR("t7_xxhash_idx")] [VARCHAR("t8")] [VARCHAR("vstream_test")]]` + want := `[[VARCHAR("aggr_test")] [VARCHAR("dual")] [VARCHAR("t1")] [VARCHAR("t1_id2_idx")] [VARCHAR("t2")] [VARCHAR("t2_id4_idx")] [VARCHAR("t3")] [VARCHAR("t3_id7_idx")] [VARCHAR("t4")] [VARCHAR("t4_id2_idx")] [VARCHAR("t5_null_vindex")] [VARCHAR("t6")] [VARCHAR("t6_id2_idx")] [VARCHAR("t7_fk")] [VARCHAR("t7_xxhash")] [VARCHAR("t7_xxhash_idx")] [VARCHAR("t8")] [VARCHAR("t9")] [VARCHAR("vstream_test")]]` assert.Equal(t, want, got) + + // DML on existing table + exec(t, conn, "insert into t9(id9, testId) values(0,0),(1,1)") // insert initial data + result := exec(t, conn, `select id9 from t9 limit 100`) // select + assert.Equal(t, 2, len(result.Rows)) + + exec(t, conn, "update t9 set testId = 42 where testId = 0") // update + assertMatches(t, conn, "select testId from t9", `[[INT64(42)] [INT64(1)]]`) + + exec(t, conn, "delete from t9 where testId = 42") // delete + assertMatches(t, conn, "select testId from t9", `[[INT64(1)]]`) + + assertMatches(t, conn, "select testId from t9 where testId = 1", `[[INT64(1)]]`) // select with WHERE clause } func TestVSchemaTrackedForNewTables(t *testing.T) { @@ -685,13 +698,21 @@ func TestVSchemaTrackedForNewTables(t *testing.T) { exec(t, conn, `create table new_table_tracked(id bigint, name varchar(100), primary key(id)) Engine=InnoDB`) // wait for vttablet's schema reload interval to pass - time.Sleep(2 * time.Second) + time.Sleep(5 * time.Second) // check if the new table is part of the schema - qr := exec(t, conn, "SHOW VSCHEMA TABLES") - got := fmt.Sprintf("%v", qr.Rows) - want := `[[VARCHAR("aggr_test")] [VARCHAR("dual")] [VARCHAR("new_table_tracked")] [VARCHAR("t1")] [VARCHAR("t1_id2_idx")] [VARCHAR("t2")] [VARCHAR("t2_id4_idx")] [VARCHAR("t3")] [VARCHAR("t3_id7_idx")] [VARCHAR("t4")] [VARCHAR("t4_id2_idx")] [VARCHAR("t5_null_vindex")] [VARCHAR("t6")] [VARCHAR("t6_id2_idx")] [VARCHAR("t7_fk")] [VARCHAR("t7_xxhash")] [VARCHAR("t7_xxhash_idx")] [VARCHAR("t8")] [VARCHAR("vstream_test")]]` - assert.Equal(t, want, got) + assertMatches(t, conn, "SHOW VSCHEMA TABLES", `[[VARCHAR("aggr_test")] [VARCHAR("dual")] [VARCHAR("new_table_tracked")] [VARCHAR("t1")] [VARCHAR("t1_id2_idx")] [VARCHAR("t2")] [VARCHAR("t2_id4_idx")] [VARCHAR("t3")] [VARCHAR("t3_id7_idx")] [VARCHAR("t4")] [VARCHAR("t4_id2_idx")] [VARCHAR("t5_null_vindex")] [VARCHAR("t6")] [VARCHAR("t6_id2_idx")] [VARCHAR("t7_fk")] [VARCHAR("t7_xxhash")] [VARCHAR("t7_xxhash_idx")] [VARCHAR("t8")] [VARCHAR("t9")] [VARCHAR("vstream_test")]]`) + + // DML on new table + exec(t, conn, `insert into new_table_tracked(id) values(0),(1)`) // insert initial data + assertMatches(t, conn, "select id from new_table_tracked", `[[INT64(0)] [INT64(1)]]`) // select + assertMatches(t, conn, "select id from new_table_tracked where id = 1 ", `[[INT64(1)]]`) // select with WHERE clause + + exec(t, conn, `update new_table_tracked set name = "newName1" where id = 1`) // update + assertMatches(t, conn, "select name from new_table_tracked where id = 1 ", `[[VARCHAR("newName1")]]`) + + exec(t, conn, "delete from new_table_tracked where id = 0") // delete + assertMatches(t, conn, "select id from new_table_tracked", `[[INT64(1)]]`) } func assertMatches(t *testing.T, conn *mysql.Conn, query, expected string) { diff --git a/go/test/endtoend/vtgate/schematracker/schematracker_test.go b/go/test/endtoend/vtgate/schematracker/schematracker_test.go index f114c76b8c..dbc9dab3de 100644 --- a/go/test/endtoend/vtgate/schematracker/schematracker_test.go +++ b/go/test/endtoend/vtgate/schematracker/schematracker_test.go @@ -62,7 +62,7 @@ func TestNewUnshardedTable(t *testing.T) { } // enabling and setting the schema reload time to one second - clusterInstance.VtTabletExtraArgs = []string{"-queryserver-config-schema-change-signal", "-queryserver-config-schema-change-signal-interval", "1"} + clusterInstance.VtTabletExtraArgs = []string{"-queryserver-config-schema-change-signal", "-queryserver-config-schema-change-signal-interval", "0.1"} err = clusterInstance.StartUnshardedKeyspace(*keyspace, 1, false) require.NoError(t, err) @@ -90,7 +90,7 @@ func TestNewUnshardedTable(t *testing.T) { exec(t, conn, `create table new_table_tracked(id bigint, name varchar(100), primary key(id)) Engine=InnoDB`) // waiting for the vttablet's schema_reload interval to kick in - time.Sleep(2 * time.Second) + time.Sleep(5 * time.Second) // ensuring our new table is in the schema qr = exec(t, conn, "SHOW VSCHEMA TABLES") From 3f6b7a61fead5123827303b827c9c9b7f4b1f2d7 Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Thu, 3 Jun 2021 19:51:40 +0200 Subject: [PATCH 17/64] attempt to fix panic on DML against sharded keyspace Signed-off-by: Florent Poinsard --- go/test/endtoend/vtgate/misc_test.go | 1 + go/vt/vtgate/engine/insert.go | 3 +++ 2 files changed, 4 insertions(+) diff --git a/go/test/endtoend/vtgate/misc_test.go b/go/test/endtoend/vtgate/misc_test.go index 32b4f3b713..2838aecd25 100644 --- a/go/test/endtoend/vtgate/misc_test.go +++ b/go/test/endtoend/vtgate/misc_test.go @@ -704,6 +704,7 @@ func TestVSchemaTrackedForNewTables(t *testing.T) { assertMatches(t, conn, "SHOW VSCHEMA TABLES", `[[VARCHAR("aggr_test")] [VARCHAR("dual")] [VARCHAR("new_table_tracked")] [VARCHAR("t1")] [VARCHAR("t1_id2_idx")] [VARCHAR("t2")] [VARCHAR("t2_id4_idx")] [VARCHAR("t3")] [VARCHAR("t3_id7_idx")] [VARCHAR("t4")] [VARCHAR("t4_id2_idx")] [VARCHAR("t5_null_vindex")] [VARCHAR("t6")] [VARCHAR("t6_id2_idx")] [VARCHAR("t7_fk")] [VARCHAR("t7_xxhash")] [VARCHAR("t7_xxhash_idx")] [VARCHAR("t8")] [VARCHAR("t9")] [VARCHAR("vstream_test")]]`) // DML on new table + assertMatches(t, conn, "select id from new_table_tracked", `[]`) // select exec(t, conn, `insert into new_table_tracked(id) values(0),(1)`) // insert initial data assertMatches(t, conn, "select id from new_table_tracked", `[[INT64(0)] [INT64(1)]]`) // select assertMatches(t, conn, "select id from new_table_tracked where id = 1 ", `[[INT64(1)]]`) // select with WHERE clause diff --git a/go/vt/vtgate/engine/insert.go b/go/vt/vtgate/engine/insert.go index d09be70846..4774cad2a6 100644 --- a/go/vt/vtgate/engine/insert.go +++ b/go/vt/vtgate/engine/insert.go @@ -394,6 +394,9 @@ func (ins *Insert) getInsertShardedRoute(vcursor VCursor, bindVars map[string]*q // keyspace ids. For regular inserts, a failure to find a route // results in an error. For 'ignore' type inserts, the keyspace // id is returned as nil, which is used later to drop the corresponding rows. + if len(vindexRowsValues) == 0 || len(ins.Table.ColumnVindexes) == 0 { + return nil, nil, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "[BUG] table without a primary vindex is not expectedd") + } keyspaceIDs, err := ins.processPrimary(vcursor, vindexRowsValues[0], ins.Table.ColumnVindexes[0]) if err != nil { return nil, nil, err From f0590feeddc303997a589580c0aef3ede0bb6346 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Mon, 7 Jun 2021 10:33:29 +0200 Subject: [PATCH 18/64] Remove duplication Signed-off-by: Rohit Nayak --- go/vt/wrangler/materializer.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/go/vt/wrangler/materializer.go b/go/vt/wrangler/materializer.go index 1460076131..0418df15a1 100644 --- a/go/vt/wrangler/materializer.go +++ b/go/vt/wrangler/materializer.go @@ -1068,10 +1068,7 @@ func (mz *materializer) generateInserts(ctx context.Context) (string, error) { } else { sel.Where = &sqlparser.Where{ Type: sqlparser.WhereClause, - Expr: &sqlparser.FuncExpr{ - Name: sqlparser.NewColIdent("in_keyrange"), - Exprs: subExprs, - }, + Expr: inKeyRange, } } From 5b5ccd83963e80d884b46ca65b9e59f277ea70e6 Mon Sep 17 00:00:00 2001 From: Vicent Marti Date: Fri, 28 May 2021 11:46:51 +0200 Subject: [PATCH 19/64] proto: enable pooling for vreplication Signed-off-by: Vicent Marti --- Makefile | 4 +- .../proto/binlogdata/binlogdata_vtproto.pb.go | 59 +++++++++++++++++-- go/vt/proto/query/query_vtproto.pb.go | 26 +++++++- go/vt/vttablet/grpcqueryservice/server.go | 4 +- go/vt/vttablet/grpctabletconn/conn.go | 8 +-- .../tabletmanager/vreplication/vcopier.go | 4 +- 6 files changed, 91 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index dae1a8e271..46aa5673d3 100644 --- a/Makefile +++ b/Makefile @@ -232,7 +232,9 @@ $(PROTO_GO_OUTS): minimaltools install_protoc-gen-go proto/*.proto --go_out=. --plugin protoc-gen-go="${GOBIN}/protoc-gen-go" \ --go-grpc_out=. --plugin protoc-gen-go-grpc="${GOBIN}/protoc-gen-go-grpc" \ --go-vtproto_out=. --plugin protoc-gen-go-vtproto="${GOBIN}/protoc-gen-go-vtproto" \ - --go-vtproto_opt=features=marshal+unmarshal+size \ + --go-vtproto_opt=features=marshal+unmarshal+size+pool \ + --go-vtproto_opt=pool=vitess.io/vitess/go/vt/proto/query.Row \ + --go-vtproto_opt=pool=vitess.io/vitess/go/vt/proto/binlogdata.VStreamRowsResponse \ -I${PWD}/dist/vt-protoc-3.6.1/include:proto proto/$${name}.proto; \ done cp -Rf vitess.io/vitess/go/vt/proto/* go/vt/proto diff --git a/go/vt/proto/binlogdata/binlogdata_vtproto.pb.go b/go/vt/proto/binlogdata/binlogdata_vtproto.pb.go index 3c2cea1992..194ce30aaf 100644 --- a/go/vt/proto/binlogdata/binlogdata_vtproto.pb.go +++ b/go/vt/proto/binlogdata/binlogdata_vtproto.pb.go @@ -9,6 +9,7 @@ import ( protoimpl "google.golang.org/protobuf/runtime/protoimpl" io "io" bits "math/bits" + sync "sync" query "vitess.io/vitess/go/vt/proto/query" topodata "vitess.io/vitess/go/vt/proto/topodata" vtrpc "vitess.io/vitess/go/vt/proto/vtrpc" @@ -1884,6 +1885,35 @@ func encodeVarint(dAtA []byte, offset int, v uint64) int { dAtA[offset] = uint8(v) return base } + +var vtprotoPool_VStreamRowsResponse = sync.Pool{ + New: func() interface{} { + return &VStreamRowsResponse{} + }, +} + +func (m *VStreamRowsResponse) ResetVT() { + f0 := m.Fields[:0] + f1 := m.Pkfields[:0] + for _, mm := range m.Rows { + mm.ResetVT() + } + f2 := m.Rows[:0] + m.Lastpk.ReturnToVTPool() + m.Reset() + m.Fields = f0 + m.Pkfields = f1 + m.Rows = f2 +} +func (m *VStreamRowsResponse) ReturnToVTPool() { + if m != nil { + m.ResetVT() + vtprotoPool_VStreamRowsResponse.Put(m) + } +} +func VStreamRowsResponseFromVTPool() *VStreamRowsResponse { + return vtprotoPool_VStreamRowsResponse.Get().(*VStreamRowsResponse) +} func (m *Charset) SizeVT() (n int) { if m == nil { return 0 @@ -6322,7 +6352,14 @@ func (m *VStreamRowsResponse) UnmarshalVT(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Fields = append(m.Fields, &query.Field{}) + if len(m.Fields) == cap(m.Fields) { + m.Fields = append(m.Fields, &query.Field{}) + } else { + m.Fields = m.Fields[:len(m.Fields)+1] + if m.Fields[len(m.Fields)-1] == nil { + m.Fields[len(m.Fields)-1] = &query.Field{} + } + } if err := m.Fields[len(m.Fields)-1].UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { return err } @@ -6356,7 +6393,14 @@ func (m *VStreamRowsResponse) UnmarshalVT(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Pkfields = append(m.Pkfields, &query.Field{}) + if len(m.Pkfields) == cap(m.Pkfields) { + m.Pkfields = append(m.Pkfields, &query.Field{}) + } else { + m.Pkfields = m.Pkfields[:len(m.Pkfields)+1] + if m.Pkfields[len(m.Pkfields)-1] == nil { + m.Pkfields[len(m.Pkfields)-1] = &query.Field{} + } + } if err := m.Pkfields[len(m.Pkfields)-1].UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { return err } @@ -6422,7 +6466,14 @@ func (m *VStreamRowsResponse) UnmarshalVT(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Rows = append(m.Rows, &query.Row{}) + if len(m.Rows) == cap(m.Rows) { + m.Rows = append(m.Rows, &query.Row{}) + } else { + m.Rows = m.Rows[:len(m.Rows)+1] + if m.Rows[len(m.Rows)-1] == nil { + m.Rows[len(m.Rows)-1] = &query.Row{} + } + } if err := m.Rows[len(m.Rows)-1].UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { return err } @@ -6457,7 +6508,7 @@ func (m *VStreamRowsResponse) UnmarshalVT(dAtA []byte) error { return io.ErrUnexpectedEOF } if m.Lastpk == nil { - m.Lastpk = &query.Row{} + m.Lastpk = query.RowFromVTPool() } if err := m.Lastpk.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { return err diff --git a/go/vt/proto/query/query_vtproto.pb.go b/go/vt/proto/query/query_vtproto.pb.go index b77b9e8e27..615a71e3d7 100644 --- a/go/vt/proto/query/query_vtproto.pb.go +++ b/go/vt/proto/query/query_vtproto.pb.go @@ -11,6 +11,7 @@ import ( io "io" math "math" bits "math/bits" + sync "sync" topodata "vitess.io/vitess/go/vt/proto/topodata" vtrpc "vitess.io/vitess/go/vt/proto/vtrpc" ) @@ -4045,6 +4046,29 @@ func encodeVarint(dAtA []byte, offset int, v uint64) int { dAtA[offset] = uint8(v) return base } + +var vtprotoPool_Row = sync.Pool{ + New: func() interface{} { + return &Row{} + }, +} + +func (m *Row) ResetVT() { + f0 := m.Lengths[:0] + f1 := m.Values[:0] + m.Reset() + m.Lengths = f0 + m.Values = f1 +} +func (m *Row) ReturnToVTPool() { + if m != nil { + m.ResetVT() + vtprotoPool_Row.Put(m) + } +} +func RowFromVTPool() *Row { + return vtprotoPool_Row.Get().(*Row) +} func (m *Target) SizeVT() (n int) { if m == nil { return 0 @@ -7100,7 +7124,7 @@ func (m *Row) UnmarshalVT(dAtA []byte) error { } } elementCount = count - if elementCount != 0 && len(m.Lengths) == 0 { + if elementCount != 0 && len(m.Lengths) == 0 && cap(m.Lengths) < elementCount { m.Lengths = make([]int64, 0, elementCount) } for iNdEx < postIndex { diff --git a/go/vt/vttablet/grpcqueryservice/server.go b/go/vt/vttablet/grpcqueryservice/server.go index 15218c8f24..a4fb21b5ea 100644 --- a/go/vt/vttablet/grpcqueryservice/server.go +++ b/go/vt/vttablet/grpcqueryservice/server.go @@ -17,10 +17,10 @@ limitations under the License. package grpcqueryservice import ( - "google.golang.org/grpc" - "context" + "google.golang.org/grpc" + "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/callerid" "vitess.io/vitess/go/vt/callinfo" diff --git a/go/vt/vttablet/grpctabletconn/conn.go b/go/vt/vttablet/grpctabletconn/conn.go index 802345bcd9..e3a5ae50c6 100644 --- a/go/vt/vttablet/grpctabletconn/conn.go +++ b/go/vt/vttablet/grpctabletconn/conn.go @@ -675,18 +675,18 @@ func (conn *gRPCQueryClient) VStreamRows(ctx context.Context, target *querypb.Ta return err } for { - r, err := stream.Recv() + r := binlogdatapb.VStreamRowsResponseFromVTPool() + err := stream.RecvMsg(r) if err != nil { return tabletconn.ErrorFromGRPC(err) } - select { - case <-ctx.Done(): + if ctx.Err() != nil { return ctx.Err() - default: } if err := send(r); err != nil { return err } + r.ReturnToVTPool() } } diff --git a/go/vt/vttablet/tabletmanager/vreplication/vcopier.go b/go/vt/vttablet/tabletmanager/vreplication/vcopier.go index a16cb1ba52..aeff052e01 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vcopier.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vcopier.go @@ -244,13 +244,13 @@ func (vc *vcopier) copyTable(ctx context.Context, tableName string, copyState ma } fieldEvent := &binlogdatapb.FieldEvent{ TableName: initialPlan.SendRule.Match, - Fields: rows.Fields, } + fieldEvent.Fields = append(fieldEvent.Fields, rows.Fields...) vc.tablePlan, err = plan.buildExecutionPlan(fieldEvent) if err != nil { return err } - pkfields = rows.Pkfields + pkfields = append(pkfields, rows.Pkfields...) buf := sqlparser.NewTrackedBuffer(nil) buf.Myprintf("update _vt.copy_state set lastpk=%a where vrepl_id=%s and table_name=%s", ":lastpk", strconv.Itoa(int(vc.vr.id)), encodeString(tableName)) updateCopyState = buf.ParsedQuery() From 2399ee19c69490dd0dd6b8830f4e0712e378e981 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Thu, 13 May 2021 15:11:40 +0200 Subject: [PATCH 20/64] Refactor PKInfoMap to get info for all columns in the table as ColInfoMap as a precursor to adding Extra info for detecting generated columns Signed-off-by: Rohit Nayak --- .../vreplication/replicator_plan.go | 12 +-- .../vreplication/replicator_plan_test.go | 16 ++-- .../vreplication/table_plan_builder.go | 67 +++++++++-------- .../tabletmanager/vreplication/vcopier.go | 4 +- .../tabletmanager/vreplication/vplayer.go | 2 +- .../tabletmanager/vreplication/vreplicator.go | 74 ++++++++++--------- 6 files changed, 91 insertions(+), 84 deletions(-) diff --git a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go index 672c40c56b..385e90c0f2 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go +++ b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go @@ -53,7 +53,7 @@ type ReplicatorPlan struct { VStreamFilter *binlogdatapb.Filter TargetTables map[string]*TablePlan TablePlans map[string]*TablePlan - PKInfoMap map[string][]*PrimaryKeyInfo + ColInfoMap map[string][]*ColumnInfo stats *binlogplayer.Stats } @@ -94,10 +94,10 @@ func (rp *ReplicatorPlan) buildExecutionPlan(fieldEvent *binlogdatapb.FieldEvent // requires us to wait for the field info sent by the source. func (rp *ReplicatorPlan) buildFromFields(tableName string, lastpk *sqltypes.Result, fields []*querypb.Field) (*TablePlan, error) { tpb := &tablePlanBuilder{ - name: sqlparser.NewTableIdent(tableName), - lastpk: lastpk, - pkInfos: rp.PKInfoMap[tableName], - stats: rp.stats, + name: sqlparser.NewTableIdent(tableName), + lastpk: lastpk, + columnInfos: rp.ColInfoMap[tableName], + stats: rp.stats, } for _, field := range fields { colName := sqlparser.NewColIdent(field.Name) @@ -114,7 +114,7 @@ func (rp *ReplicatorPlan) buildFromFields(tableName string, lastpk *sqltypes.Res tpb.colExprs = append(tpb.colExprs, cexpr) } // The following actions are a subset of buildTablePlan. - if err := tpb.analyzePK(rp.PKInfoMap); err != nil { + if err := tpb.analyzePK(rp.ColInfoMap); err != nil { return nil, err } return tpb.generate(), nil diff --git a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan_test.go b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan_test.go index 6297ac0652..fc1736dc5b 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan_test.go @@ -669,8 +669,8 @@ func TestBuildPlayerPlan(t *testing.T) { err: "group by expression is not allowed to reference an aggregate expression: a", }} - PrimaryKeyInfos := map[string][]*PrimaryKeyInfo{ - "t1": {&PrimaryKeyInfo{Name: "c1"}}, + PrimaryKeyInfos := map[string][]*ColumnInfo{ + "t1": {&ColumnInfo{Name: "c1", IsPK: true}}, } copyState := map[string]*sqltypes.Result{ @@ -711,9 +711,9 @@ func TestBuildPlayerPlan(t *testing.T) { } func TestBuildPlayerPlanNoDup(t *testing.T) { - PrimaryKeyInfos := map[string][]*PrimaryKeyInfo{ - "t1": {&PrimaryKeyInfo{Name: "c1"}}, - "t2": {&PrimaryKeyInfo{Name: "c2"}}, + PrimaryKeyInfos := map[string][]*ColumnInfo{ + "t1": {&ColumnInfo{Name: "c1"}}, + "t2": {&ColumnInfo{Name: "c2"}}, } input := &binlogdatapb.Filter{ Rules: []*binlogdatapb.Rule{{ @@ -732,9 +732,9 @@ func TestBuildPlayerPlanNoDup(t *testing.T) { } func TestBuildPlayerPlanExclude(t *testing.T) { - PrimaryKeyInfos := map[string][]*PrimaryKeyInfo{ - "t1": {&PrimaryKeyInfo{Name: "c1"}}, - "t2": {&PrimaryKeyInfo{Name: "c2"}}, + PrimaryKeyInfos := map[string][]*ColumnInfo{ + "t1": {&ColumnInfo{Name: "c1"}}, + "t2": {&ColumnInfo{Name: "c2"}}, } input := &binlogdatapb.Filter{ Rules: []*binlogdatapb.Rule{{ diff --git a/go/vt/vttablet/tabletmanager/vreplication/table_plan_builder.go b/go/vt/vttablet/tabletmanager/vreplication/table_plan_builder.go index 66c57ca09d..8ec80bc7a2 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/table_plan_builder.go +++ b/go/vt/vttablet/tabletmanager/vreplication/table_plan_builder.go @@ -47,13 +47,13 @@ type tablePlanBuilder struct { // selColumns keeps track of the columns we want to pull from source. // If Lastpk is set, we compare this list against the table's pk and // add missing references. - selColumns map[string]bool - colExprs []*colExpr - onInsert insertType - pkCols []*colExpr - lastpk *sqltypes.Result - pkInfos []*PrimaryKeyInfo - stats *binlogplayer.Stats + selColumns map[string]bool + colExprs []*colExpr + onInsert insertType + pkCols []*colExpr + lastpk *sqltypes.Result + columnInfos []*ColumnInfo + stats *binlogplayer.Stats } // colExpr describes the processing to be performed to @@ -112,7 +112,7 @@ const ( // a table-specific rule is built to be sent to the source. We don't send the // original rule to the source because it may not match the same tables as the // target. -// pkInfoMap specifies the list of primary key columns for each table. +// colInfoMap specifies the list of primary key columns for each table. // copyState is a map of tables that have not been fully copied yet. // If a table is not present in copyState, then it has been fully copied. If so, // all replication events are applied. The table still has to match a Filter.Rule. @@ -123,15 +123,15 @@ const ( // The TablePlan built is a partial plan. The full plan for a table is built // when we receive field information from events or rows sent by the source. // buildExecutionPlan is the function that builds the full plan. -func buildReplicatorPlan(filter *binlogdatapb.Filter, pkInfoMap map[string][]*PrimaryKeyInfo, copyState map[string]*sqltypes.Result, stats *binlogplayer.Stats) (*ReplicatorPlan, error) { +func buildReplicatorPlan(filter *binlogdatapb.Filter, colInfoMap map[string][]*ColumnInfo, copyState map[string]*sqltypes.Result, stats *binlogplayer.Stats) (*ReplicatorPlan, error) { plan := &ReplicatorPlan{ VStreamFilter: &binlogdatapb.Filter{FieldEventMode: filter.FieldEventMode}, TargetTables: make(map[string]*TablePlan), TablePlans: make(map[string]*TablePlan), - PKInfoMap: pkInfoMap, + ColInfoMap: colInfoMap, stats: stats, } - for tableName := range pkInfoMap { + for tableName := range colInfoMap { lastpk, ok := copyState[tableName] if ok && lastpk == nil { // Don't replicate uncopied tables. @@ -144,7 +144,7 @@ func buildReplicatorPlan(filter *binlogdatapb.Filter, pkInfoMap map[string][]*Pr if rule == nil { continue } - tablePlan, err := buildTablePlan(tableName, rule.Filter, pkInfoMap, lastpk, stats) + tablePlan, err := buildTablePlan(tableName, rule.Filter, colInfoMap, lastpk, stats) if err != nil { return nil, err } @@ -183,7 +183,7 @@ func MatchTable(tableName string, filter *binlogdatapb.Filter) (*binlogdatapb.Ru return nil, nil } -func buildTablePlan(tableName, filter string, pkInfoMap map[string][]*PrimaryKeyInfo, lastpk *sqltypes.Result, stats *binlogplayer.Stats) (*TablePlan, error) { +func buildTablePlan(tableName, filter string, colInfoMap map[string][]*ColumnInfo, lastpk *sqltypes.Result, stats *binlogplayer.Stats) (*TablePlan, error) { query := filter // generate equivalent select statement if filter is empty or a keyrange. switch { @@ -231,10 +231,10 @@ func buildTablePlan(tableName, filter string, pkInfoMap map[string][]*PrimaryKey From: sel.From, Where: sel.Where, }, - selColumns: make(map[string]bool), - lastpk: lastpk, - pkInfos: pkInfoMap[tableName], - stats: stats, + selColumns: make(map[string]bool), + lastpk: lastpk, + columnInfos: colInfoMap[tableName], + stats: stats, } if err := tpb.analyzeExprs(sel.SelectExprs); err != nil { @@ -255,7 +255,7 @@ func buildTablePlan(tableName, filter string, pkInfoMap map[string][]*PrimaryKey if err := tpb.analyzeGroupBy(sel.GroupBy); err != nil { return nil, err } - if err := tpb.analyzePK(pkInfoMap); err != nil { + if err := tpb.analyzePK(colInfoMap); err != nil { return nil, err } @@ -475,22 +475,25 @@ func (tpb *tablePlanBuilder) analyzeGroupBy(groupBy sqlparser.GroupBy) error { } // analyzePK builds tpb.pkCols. -func (tpb *tablePlanBuilder) analyzePK(pkInfoMap map[string][]*PrimaryKeyInfo) error { - pkcols, ok := pkInfoMap[tpb.name.String()] +func (tpb *tablePlanBuilder) analyzePK(colInfoMap map[string][]*ColumnInfo) error { + cols, ok := colInfoMap[tpb.name.String()] if !ok { return fmt.Errorf("table %s not found in schema", tpb.name) } - for _, pkcol := range pkcols { - cexpr := tpb.findCol(sqlparser.NewColIdent(pkcol.Name)) + for _, col := range cols { + if !col.IsPK { + continue + } + cexpr := tpb.findCol(sqlparser.NewColIdent(col.Name)) if cexpr == nil { - return fmt.Errorf("primary key column %v not found in select list", pkcol) + return fmt.Errorf("primary key column %v not found in select list", col) } if cexpr.operation != opExpr { - return fmt.Errorf("primary key column %v is not allowed to reference an aggregate expression", pkcol) + return fmt.Errorf("primary key column %v is not allowed to reference an aggregate expression", col) } cexpr.isPK = true - cexpr.dataType = pkcol.DataType - cexpr.columnType = pkcol.ColumnType + cexpr.dataType = col.DataType + cexpr.columnType = col.ColumnType tpb.pkCols = append(tpb.pkCols, cexpr) } return nil @@ -708,13 +711,13 @@ func (tpb *tablePlanBuilder) generateWhere(buf *sqlparser.TrackedBuffer, bvf *bi } func (tpb *tablePlanBuilder) getCharsetAndCollation(pkname string) (charSet string, collation string) { - for _, pkInfo := range tpb.pkInfos { - if strings.EqualFold(pkInfo.Name, pkname) { - if pkInfo.CharSet != "" { - charSet = fmt.Sprintf(" _%s ", pkInfo.CharSet) + for _, colInfo := range tpb.columnInfos { + if colInfo.IsPK && strings.EqualFold(colInfo.Name, pkname) { + if colInfo.CharSet != "" { + charSet = fmt.Sprintf(" _%s ", colInfo.CharSet) } - if pkInfo.Collation != "" { - collation = fmt.Sprintf(" COLLATE %s ", pkInfo.Collation) + if colInfo.Collation != "" { + collation = fmt.Sprintf(" COLLATE %s ", colInfo.Collation) } } } diff --git a/go/vt/vttablet/tabletmanager/vreplication/vcopier.go b/go/vt/vttablet/tabletmanager/vreplication/vcopier.go index a16cb1ba52..7975dc3c05 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vcopier.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vcopier.go @@ -57,7 +57,7 @@ func newVCopier(vr *vreplicator) *vcopier { func (vc *vcopier) initTablesForCopy(ctx context.Context) error { defer vc.vr.dbClient.Rollback() - plan, err := buildReplicatorPlan(vc.vr.source.Filter, vc.vr.pkInfoMap, nil, vc.vr.stats) + plan, err := buildReplicatorPlan(vc.vr.source.Filter, vc.vr.colInfoMap, nil, vc.vr.stats) if err != nil { return err } @@ -200,7 +200,7 @@ func (vc *vcopier) copyTable(ctx context.Context, tableName string, copyState ma log.Infof("Copying table %s, lastpk: %v", tableName, copyState[tableName]) - plan, err := buildReplicatorPlan(vc.vr.source.Filter, vc.vr.pkInfoMap, nil, vc.vr.stats) + plan, err := buildReplicatorPlan(vc.vr.source.Filter, vc.vr.colInfoMap, nil, vc.vr.stats) if err != nil { return err } diff --git a/go/vt/vttablet/tabletmanager/vreplication/vplayer.go b/go/vt/vttablet/tabletmanager/vreplication/vplayer.go index 9933d0db7d..c62217429c 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vplayer.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vplayer.go @@ -105,7 +105,7 @@ func (vp *vplayer) play(ctx context.Context) error { return nil } - plan, err := buildReplicatorPlan(vp.vr.source.Filter, vp.vr.pkInfoMap, vp.copyState, vp.vr.stats) + plan, err := buildReplicatorPlan(vp.vr.source.Filter, vp.vr.colInfoMap, vp.copyState, vp.vr.stats) if err != nil { vp.vr.stats.ErrorCounts.Add([]string{"Plan"}, 1) return err diff --git a/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go b/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go index 9a9e8de1ff..28cfadb672 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go @@ -78,8 +78,8 @@ type vreplicator struct { state string stats *binlogplayer.Stats // mysqld is used to fetch the local schema. - mysqld mysqlctl.MysqlDaemon - pkInfoMap map[string][]*PrimaryKeyInfo + mysqld mysqlctl.MysqlDaemon + colInfoMap map[string][]*ColumnInfo originalFKCheckSetting int64 } @@ -154,11 +154,11 @@ func (vr *vreplicator) Replicate(ctx context.Context) error { } func (vr *vreplicator) replicate(ctx context.Context) error { - pkInfo, err := vr.buildPkInfoMap(ctx) + colInfo, err := vr.buildColInfoMap(ctx) if err != nil { return err } - vr.pkInfoMap = pkInfo + vr.colInfoMap = colInfo if err := vr.getSettingFKCheck(); err != nil { return err } @@ -224,22 +224,23 @@ func (vr *vreplicator) replicate(ctx context.Context) error { } } -// PrimaryKeyInfo is used to store charset and collation for primary keys where applicable -type PrimaryKeyInfo struct { +// ColumnInfo is used to store charset and collation for primary keys where applicable +type ColumnInfo struct { Name string CharSet string Collation string DataType string ColumnType string + IsPK bool } -func (vr *vreplicator) buildPkInfoMap(ctx context.Context) (map[string][]*PrimaryKeyInfo, error) { +func (vr *vreplicator) buildColInfoMap(ctx context.Context) (map[string][]*ColumnInfo, error) { schema, err := vr.mysqld.GetSchema(ctx, vr.dbClient.DBName(), []string{"/.*/"}, nil, false) if err != nil { return nil, err } queryTemplate := "select character_set_name, collation_name, column_name, data_type, column_type from information_schema.columns where table_schema=%s and table_name=%s;" - pkInfoMap := make(map[string][]*PrimaryKeyInfo) + colInfoMap := make(map[string][]*ColumnInfo) for _, td := range schema.TableDefinitions { query := fmt.Sprintf(queryTemplate, encodeString(vr.dbClient.DBName()), encodeString(td.Name)) @@ -257,47 +258,50 @@ func (vr *vreplicator) buildPkInfoMap(ctx context.Context) (map[string][]*Primar } else { pks = td.Columns } - var pkInfos []*PrimaryKeyInfo - for _, pk := range pks { + var colInfo []*ColumnInfo + for _, row := range qr.Rows { charSet := "" collation := "" + columnName := "" + isPK := false var dataType, columnType string - for _, row := range qr.Rows { - columnName := row[2].ToString() - if strings.EqualFold(columnName, pk) { - var currentField *querypb.Field - for _, field := range td.Fields { - if field.Name == pk { - currentField = field - break - } - } - if currentField == nil { - continue - } - dataType = row[3].ToString() - columnType = row[4].ToString() - if sqltypes.IsText(currentField.Type) { - charSet = row[0].ToString() - collation = row[1].ToString() - } + columnName = row[2].ToString() + var currentField *querypb.Field + for _, field := range td.Fields { + if field.Name == columnName { + currentField = field break } } - if dataType == "" || columnType == "" { - return nil, fmt.Errorf("no dataType/columnType found in information_schema.columns for table %s, column %s", td.Name, pk) + if currentField == nil { + continue } - pkInfos = append(pkInfos, &PrimaryKeyInfo{ - Name: pk, + dataType = row[3].ToString() + columnType = row[4].ToString() + if sqltypes.IsText(currentField.Type) { + charSet = row[0].ToString() + collation = row[1].ToString() + } + if dataType == "" || columnType == "" { + return nil, fmt.Errorf("no dataType/columnType found in information_schema.columns for table %s, column %s", td.Name, columnName) + } + for _, pk := range pks { + if columnName == pk { + isPK = true + } + } + colInfo = append(colInfo, &ColumnInfo{ + Name: columnName, CharSet: charSet, Collation: collation, DataType: dataType, ColumnType: columnType, + IsPK: isPK, }) } - pkInfoMap[td.Name] = pkInfos + colInfoMap[td.Name] = colInfo } - return pkInfoMap, nil + return colInfoMap, nil } func (vr *vreplicator) readSettings(ctx context.Context) (settings binlogplayer.VRSettings, numTablesToCopy int64, err error) { From 32d438eec7572be6c00c5aa7a5fdd5b5a3986973 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Thu, 13 May 2021 21:26:05 +0200 Subject: [PATCH 21/64] Ignore inserting/updating generated columns on the target. Add e2e tests to test generated columns in source as well as target for reshard and materialize Signed-off-by: Rohit Nayak --- go/test/endtoend/vreplication/config.go | 6 ++-- .../vreplication/unsharded_init_data.sql | 6 ++-- .../vreplication/replicator_plan.go | 13 ++++++++ .../vreplication/table_plan_builder.go | 26 +++++++++++++++ .../tabletmanager/vreplication/vreplicator.go | 33 +++++++++++-------- 5 files changed, 65 insertions(+), 19 deletions(-) diff --git a/go/test/endtoend/vreplication/config.go b/go/test/endtoend/vreplication/config.go index 214876a205..937d27bf90 100644 --- a/go/test/endtoend/vreplication/config.go +++ b/go/test/endtoend/vreplication/config.go @@ -6,7 +6,7 @@ create table product(pid int, description varbinary(128), primary key(pid)); create table customer(cid int, name varbinary(128), meta json default null, typ enum('individual','soho','enterprise'), sport set('football','cricket','baseball'),ts timestamp not null default current_timestamp, primary key(cid)) CHARSET=utf8mb4; create table customer_seq(id int, next_id bigint, cache bigint, primary key(id)) comment 'vitess_sequence'; create table merchant(mname varchar(128), category varchar(128), primary key(mname)) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; -create table orders(oid int, cid int, pid int, mname varchar(128), price int, primary key(oid)); +create table orders(oid int, cid int, pid int, mname varchar(128), price int, qty int, total int as (qty * price), total2 int as (qty * price) stored, primary key(oid)); create table order_seq(id int, next_id bigint, cache bigint, primary key(id)) comment 'vitess_sequence'; create table customer2(cid int, name varbinary(128), typ enum('individual','soho','enterprise'), sport set('football','cricket','baseball'),ts timestamp not null default current_timestamp, primary key(cid)); create table customer_seq2(id int, next_id bigint, cache bigint, primary key(id)) comment 'vitess_sequence'; @@ -243,8 +243,8 @@ create table tenant(tenant_id binary(16), name varbinary(16), primary key (tenan "targetKeyspace": "merchant", "tableSettings": [{ "targetTable": "morders", - "sourceExpression": "select * from orders", - "create_ddl": "create table morders(oid int, cid int, mname varchar(128), pid int, price int, primary key(oid))" + "sourceExpression": "select oid, cid, mname, pid, price, qty, total from orders", + "create_ddl": "create table morders(oid int, cid int, mname varchar(128), pid int, price int, qty int, total int, total2 int as (10 * total), primary key(oid))" }] } ` diff --git a/go/test/endtoend/vreplication/unsharded_init_data.sql b/go/test/endtoend/vreplication/unsharded_init_data.sql index 1b58404cfb..4dd2007243 100644 --- a/go/test/endtoend/vreplication/unsharded_init_data.sql +++ b/go/test/endtoend/vreplication/unsharded_init_data.sql @@ -5,9 +5,9 @@ insert into merchant(mname, category) values('monoprice', 'electronics'); insert into merchant(mname, category) values('newegg', 'electronics'); insert into product(pid, description) values(1, 'keyboard'); insert into product(pid, description) values(2, 'monitor'); -insert into orders(oid, cid, mname, pid, price) values(1, 1, 'monoprice', 1, 10); -insert into orders(oid, cid, mname, pid, price) values(2, 1, 'newegg', 2, 15); -insert into orders(oid, cid, mname, pid, price) values(3, 2, 'monoprice', 2, 20); +insert into orders(oid, cid, mname, pid, price, qty) values(1, 1, 'monoprice', 1, 10, 1); +insert into orders(oid, cid, mname, pid, price, qty) values(2, 1, 'newegg', 2, 15, 2); +insert into orders(oid, cid, mname, pid, price, qty) values(3, 2, 'monoprice', 2, 20, 3); insert into customer2(cid, name, typ, sport) values(1, 'john',1,'football,baseball'); insert into customer2(cid, name, typ, sport) values(2, 'paul','soho','cricket'); insert into customer2(cid, name, typ, sport) values(3, 'ringo','enterprise',''); diff --git a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go index 385e90c0f2..1eacdf2ad6 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go +++ b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go @@ -101,6 +101,19 @@ func (rp *ReplicatorPlan) buildFromFields(tableName string, lastpk *sqltypes.Res } for _, field := range fields { colName := sqlparser.NewColIdent(field.Name) + isGenerated := false + for _, colInfo := range tpb.columnInfos { + if !strings.EqualFold(colInfo.Name, field.Name) { + continue + } + if colInfo.IsGenerated { + isGenerated = true + } + break + } + if isGenerated { + continue + } cexpr := &colExpr{ colName: colName, colType: field.Type, diff --git a/go/vt/vttablet/tabletmanager/vreplication/table_plan_builder.go b/go/vt/vttablet/tabletmanager/vreplication/table_plan_builder.go index 8ec80bc7a2..cce2d93352 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/table_plan_builder.go +++ b/go/vt/vttablet/tabletmanager/vreplication/table_plan_builder.go @@ -535,6 +535,9 @@ func (tpb *tablePlanBuilder) generateInsertPart(buf *sqlparser.TrackedBuffer) *s } separator := "" for _, cexpr := range tpb.colExprs { + if tpb.isColumnGenerated(cexpr.colName) { + continue + } buf.Myprintf("%s%v", separator, cexpr.colName) separator = "," } @@ -546,6 +549,9 @@ func (tpb *tablePlanBuilder) generateValuesPart(buf *sqlparser.TrackedBuffer, bv bvf.mode = bvAfter separator := "(" for _, cexpr := range tpb.colExprs { + if tpb.isColumnGenerated(cexpr.colName) { + continue + } buf.Myprintf("%s", separator) separator = "," switch cexpr.operation { @@ -571,6 +577,9 @@ func (tpb *tablePlanBuilder) generateSelectPart(buf *sqlparser.TrackedBuffer, bv buf.WriteString(" select ") separator := "" for _, cexpr := range tpb.colExprs { + if tpb.isColumnGenerated(cexpr.colName) { + continue + } buf.Myprintf("%s", separator) separator = ", " switch cexpr.operation { @@ -604,6 +613,9 @@ func (tpb *tablePlanBuilder) generateOnDupPart(buf *sqlparser.TrackedBuffer) *sq if cexpr.isGrouped || cexpr.isPK { continue } + if tpb.isColumnGenerated(cexpr.colName) { + continue + } buf.Myprintf("%s%v=", separator, cexpr.colName) separator = ", " switch cexpr.operation { @@ -631,6 +643,9 @@ func (tpb *tablePlanBuilder) generateUpdateStatement() *sqlparser.ParsedQuery { if cexpr.isGrouped || cexpr.isPK { continue } + if tpb.isColumnGenerated(cexpr.colName) { + continue + } buf.Myprintf("%s%v=", separator, cexpr.colName) separator = ", " switch cexpr.operation { @@ -748,6 +763,17 @@ func (tpb *tablePlanBuilder) generatePKConstraint(buf *sqlparser.TrackedBuffer, buf.WriteString(")") } +func (tpb *tablePlanBuilder) isColumnGenerated(col sqlparser.ColIdent) bool { + isGenerated := false + for _, colInfo := range tpb.columnInfos { + if col.EqualString(colInfo.Name) && colInfo.IsGenerated { + isGenerated = true + break + } + } + return isGenerated +} + // bindvarFormatter is a dual mode formatter. Its behavior // can be changed dynamically changed to generate bind vars // for the 'before' row or 'after' row by setting its mode diff --git a/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go b/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go index 28cfadb672..cd827ba26e 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go @@ -226,12 +226,13 @@ func (vr *vreplicator) replicate(ctx context.Context) error { // ColumnInfo is used to store charset and collation for primary keys where applicable type ColumnInfo struct { - Name string - CharSet string - Collation string - DataType string - ColumnType string - IsPK bool + Name string + CharSet string + Collation string + DataType string + ColumnType string + IsPK bool + IsGenerated bool } func (vr *vreplicator) buildColInfoMap(ctx context.Context) (map[string][]*ColumnInfo, error) { @@ -239,7 +240,7 @@ func (vr *vreplicator) buildColInfoMap(ctx context.Context) (map[string][]*Colum if err != nil { return nil, err } - queryTemplate := "select character_set_name, collation_name, column_name, data_type, column_type from information_schema.columns where table_schema=%s and table_name=%s;" + queryTemplate := "select character_set_name, collation_name, column_name, data_type, column_type, extra from information_schema.columns where table_schema=%s and table_name=%s;" colInfoMap := make(map[string][]*ColumnInfo) for _, td := range schema.TableDefinitions { @@ -264,6 +265,7 @@ func (vr *vreplicator) buildColInfoMap(ctx context.Context) (map[string][]*Colum collation := "" columnName := "" isPK := false + isGenerated := false var dataType, columnType string columnName = row[2].ToString() var currentField *querypb.Field @@ -290,13 +292,18 @@ func (vr *vreplicator) buildColInfoMap(ctx context.Context) (map[string][]*Colum isPK = true } } + extra := row[5].ToString() + if strings.Contains(strings.ToLower(extra), "generated") { + isGenerated = true + } colInfo = append(colInfo, &ColumnInfo{ - Name: columnName, - CharSet: charSet, - Collation: collation, - DataType: dataType, - ColumnType: columnType, - IsPK: isPK, + Name: columnName, + CharSet: charSet, + Collation: collation, + DataType: dataType, + ColumnType: columnType, + IsPK: isPK, + IsGenerated: isGenerated, }) } colInfoMap[td.Name] = colInfo From 8e4c944f32de3d685bc66631185b7f01aae71488 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Fri, 14 May 2021 22:14:19 +0200 Subject: [PATCH 22/64] Update logic to ignore generated columns for additional use cases Signed-off-by: Rohit Nayak --- go/test/endtoend/vreplication/cluster.go | 5 ++ go/vt/sqlparser/parsed_query.go | 46 ++++++++--- .../vreplication/replicator_plan.go | 13 +-- .../vreplication/table_plan_builder.go | 34 +++++--- .../vreplication/vcopier_test.go | 81 +++++++++++++++++++ 5 files changed, 151 insertions(+), 28 deletions(-) diff --git a/go/test/endtoend/vreplication/cluster.go b/go/test/endtoend/vreplication/cluster.go index e9041cca28..8c90f3f9b4 100644 --- a/go/test/endtoend/vreplication/cluster.go +++ b/go/test/endtoend/vreplication/cluster.go @@ -127,6 +127,11 @@ func getClusterConfig(idx int, dataRootDir string) *ClusterConfig { } func init() { + // for local debugging set this variable so that each run uses VTDATAROOT instead of a random dir + // and also does not teardown the cluster for inspecting logs and the databases + if os.Getenv("VREPLICATION_E2E_DEBUG") != "" { + debug = true + } rand.Seed(time.Now().UTC().UnixNano()) originalVtdataroot = os.Getenv("VTDATAROOT") var mainVtDataRoot string diff --git a/go/vt/sqlparser/parsed_query.go b/go/vt/sqlparser/parsed_query.go index 5ce0581e5a..7651c4bc48 100644 --- a/go/vt/sqlparser/parsed_query.go +++ b/go/vt/sqlparser/parsed_query.go @@ -32,7 +32,7 @@ import ( ) // ParsedQuery represents a parsed query where -// bind locations are precompued for fast substitutions. +// bind locations are precomputed for fast substitutions. type ParsedQuery struct { Query string bindLocations []bindLocation @@ -86,29 +86,57 @@ func (pq *ParsedQuery) Append(buf *strings.Builder, bindVariables map[string]*qu } // AppendFromRow behaves like Append but takes a querypb.Row directly, assuming that -// the fields in the row are in the same order as the placeholders in this query. -func (pq *ParsedQuery) AppendFromRow(buf *bytes2.Buffer, fields []*querypb.Field, row *querypb.Row) error { +// the fields in the row are in the same order as the placeholders in this query. The fields might include generated +// columns which are dropped, by checking against skipFields, before binding the variables +// note: there can be more fields than bind locations since extra columns might be requested from the source if not all +// primary keys columns are not in the select list, for example. Also some values in the row may not correspond for +// values from the database on the source: sum/count for aggregation queries, for example +func (pq *ParsedQuery) AppendFromRow(buf *bytes2.Buffer, fields []*querypb.Field, row *querypb.Row, skipFields map[string]bool) error { if len(fields) < len(pq.bindLocations) { - return vterrors.Errorf(vtrpcpb.Code_INTERNAL, "wrong number of fields: got %d fields for %d bind locations ", len(fields), len(pq.bindLocations)) + return vterrors.Errorf(vtrpcpb.Code_INTERNAL, "wrong number of fields: got %d fields for %d bind locations ", + len(fields), len(pq.bindLocations)) } + + type colInfo struct { + typ querypb.Type + length int64 + offset int64 + } + rowInfo := make([]*colInfo, 0) + + offset := int64(0) + for i, field := range fields { // collect info required for fields to be bound + length := row.Lengths[i] + if !skipFields[strings.ToLower(field.Name)] { + rowInfo = append(rowInfo, &colInfo{ + typ: field.Type, + length: length, + offset: offset, + }) + } + if length > 0 { + offset += row.Lengths[i] + } + } + + // bind field values to locations var offsetQuery int - var offsetRow int64 for i, loc := range pq.bindLocations { + col := rowInfo[i] buf.WriteString(pq.Query[offsetQuery:loc.offset]) - typ := fields[i].Type + typ := col.typ if typ == querypb.Type_TUPLE { return vterrors.Errorf(vtrpcpb.Code_INTERNAL, "unexpected Type_TUPLE for value %d", i) } - length := row.Lengths[i] + length := col.length if length < 0 { // -1 means a null variable; serialize it directly buf.WriteString("null") } else { - vv := sqltypes.MakeTrusted(typ, row.Values[offsetRow:offsetRow+length]) + vv := sqltypes.MakeTrusted(typ, row.Values[col.offset:col.offset+col.length]) vv.EncodeSQLBytes2(buf) - offsetRow += length } offsetQuery = loc.offset + loc.length diff --git a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go index 1eacdf2ad6..9558533dec 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go +++ b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go @@ -94,15 +94,15 @@ func (rp *ReplicatorPlan) buildExecutionPlan(fieldEvent *binlogdatapb.FieldEvent // requires us to wait for the field info sent by the source. func (rp *ReplicatorPlan) buildFromFields(tableName string, lastpk *sqltypes.Result, fields []*querypb.Field) (*TablePlan, error) { tpb := &tablePlanBuilder{ - name: sqlparser.NewTableIdent(tableName), - lastpk: lastpk, - columnInfos: rp.ColInfoMap[tableName], - stats: rp.stats, + name: sqlparser.NewTableIdent(tableName), + lastpk: lastpk, + colInfos: rp.ColInfoMap[tableName], + stats: rp.stats, } for _, field := range fields { colName := sqlparser.NewColIdent(field.Name) isGenerated := false - for _, colInfo := range tpb.columnInfos { + for _, colInfo := range tpb.colInfos { if !strings.EqualFold(colInfo.Name, field.Name) { continue } @@ -196,6 +196,7 @@ type TablePlan struct { // a primary key column (row move). PKReferences []string Stats *binlogplayer.Stats + FieldsToSkip map[string]bool } // MarshalJSON performs a custom JSON Marshalling. @@ -233,7 +234,7 @@ func (tp *TablePlan) applyBulkInsert(sqlbuffer *bytes2.Buffer, rows *binlogdatap if i > 0 { sqlbuffer.WriteString(", ") } - if err := tp.BulkInsertValues.AppendFromRow(sqlbuffer, tp.Fields, row); err != nil { + if err := tp.BulkInsertValues.AppendFromRow(sqlbuffer, tp.Fields, row, tp.FieldsToSkip); err != nil { return nil, err } } diff --git a/go/vt/vttablet/tabletmanager/vreplication/table_plan_builder.go b/go/vt/vttablet/tabletmanager/vreplication/table_plan_builder.go index cce2d93352..38b69f1060 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/table_plan_builder.go +++ b/go/vt/vttablet/tabletmanager/vreplication/table_plan_builder.go @@ -47,13 +47,13 @@ type tablePlanBuilder struct { // selColumns keeps track of the columns we want to pull from source. // If Lastpk is set, we compare this list against the table's pk and // add missing references. - selColumns map[string]bool - colExprs []*colExpr - onInsert insertType - pkCols []*colExpr - lastpk *sqltypes.Result - columnInfos []*ColumnInfo - stats *binlogplayer.Stats + selColumns map[string]bool + colExprs []*colExpr + onInsert insertType + pkCols []*colExpr + lastpk *sqltypes.Result + colInfos []*ColumnInfo + stats *binlogplayer.Stats } // colExpr describes the processing to be performed to @@ -231,10 +231,10 @@ func buildTablePlan(tableName, filter string, colInfoMap map[string][]*ColumnInf From: sel.From, Where: sel.Where, }, - selColumns: make(map[string]bool), - lastpk: lastpk, - columnInfos: colInfoMap[tableName], - stats: stats, + selColumns: make(map[string]bool), + lastpk: lastpk, + colInfos: colInfoMap[tableName], + stats: stats, } if err := tpb.analyzeExprs(sel.SelectExprs); err != nil { @@ -295,6 +295,13 @@ func (tpb *tablePlanBuilder) generate() *TablePlan { bvf := &bindvarFormatter{} + fieldsToSkip := make(map[string]bool) + for _, colInfo := range tpb.colInfos { + if colInfo.IsGenerated { + fieldsToSkip[colInfo.Name] = true + } + } + return &TablePlan{ TargetName: tpb.name.String(), Lastpk: tpb.lastpk, @@ -306,6 +313,7 @@ func (tpb *tablePlanBuilder) generate() *TablePlan { Delete: tpb.generateDeleteStatement(), PKReferences: pkrefs, Stats: tpb.stats, + FieldsToSkip: fieldsToSkip, } } @@ -726,7 +734,7 @@ func (tpb *tablePlanBuilder) generateWhere(buf *sqlparser.TrackedBuffer, bvf *bi } func (tpb *tablePlanBuilder) getCharsetAndCollation(pkname string) (charSet string, collation string) { - for _, colInfo := range tpb.columnInfos { + for _, colInfo := range tpb.colInfos { if colInfo.IsPK && strings.EqualFold(colInfo.Name, pkname) { if colInfo.CharSet != "" { charSet = fmt.Sprintf(" _%s ", colInfo.CharSet) @@ -765,7 +773,7 @@ func (tpb *tablePlanBuilder) generatePKConstraint(buf *sqlparser.TrackedBuffer, func (tpb *tablePlanBuilder) isColumnGenerated(col sqlparser.ColIdent) bool { isGenerated := false - for _, colInfo := range tpb.columnInfos { + for _, colInfo := range tpb.colInfos { if col.EqualString(colInfo.Name) && colInfo.IsGenerated { isGenerated = true break diff --git a/go/vt/vttablet/tabletmanager/vreplication/vcopier_test.go b/go/vt/vttablet/tabletmanager/vreplication/vcopier_test.go index 326f9386bb..d2f6198222 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vcopier_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vcopier_test.go @@ -1280,3 +1280,84 @@ func TestPlayerCopyTableCancel(t *testing.T) { {"2", "bbb"}, }) } + +func TestPlayerCopyTablesWithGeneratedColumn(t *testing.T) { + flavor := strings.ToLower(env.Flavor) + // Disable tests on percona (which identifies as mysql56) and mariadb platforms in CI since they + // generated columns support was added in 5.7 and mariadb added mysql compatible generated columns in 10.2 + if strings.Contains(flavor, "56") || strings.Contains(flavor, "maria") { + return + } + defer deleteTablet(addTablet(100)) + + execStatements(t, []string{ + "create table src1(id int, val varbinary(128), val2 varbinary(128) as (concat(id, val)), val3 varbinary(128) as (concat(val, id)), id2 int, primary key(id))", + "insert into src1(id, val, id2) values(2, 'bbb', 20), (1, 'aaa', 10)", + fmt.Sprintf("create table %s.dst1(id int, val varbinary(128), val2 varbinary(128) as (concat(id, val)), val3 varbinary(128), id2 int, primary key(id))", vrepldb), + "create table src2(id int, val varbinary(128), val2 varbinary(128) as (concat(id, val)), val3 varbinary(128) as (concat(val, id)), id2 int, primary key(id))", + "insert into src2(id, val, id2) values(2, 'bbb', 20), (1, 'aaa', 10)", + fmt.Sprintf("create table %s.dst2(val3 varbinary(128), val varbinary(128), id2 int)", vrepldb), + }) + defer execStatements(t, []string{ + "drop table src1", + fmt.Sprintf("drop table %s.dst1", vrepldb), + "drop table src2", + fmt.Sprintf("drop table %s.dst2", vrepldb), + }) + env.SchemaEngine.Reload(context.Background()) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "dst1", + Filter: "select * from src1", + }, { + Match: "dst2", + Filter: "select val3, val, id2 from src2", + }}, + } + + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + query := binlogplayer.CreateVReplicationState("test", bls, "", binlogplayer.VReplicationInit, playerEngine.dbName) + qr, err := playerEngine.Exec(query) + if err != nil { + t.Fatal(err) + } + defer func() { + query := fmt.Sprintf("delete from _vt.vreplication where id = %d", qr.InsertID) + if _, err := playerEngine.Exec(query); err != nil { + t.Fatal(err) + } + expectDeleteQueries(t) + }() + + expectNontxQueries(t, []string{ + // Create the list of tables to copy and transition to Copying state. + "/insert into _vt.vreplication", + "/update _vt.vreplication set message=", + "/insert into _vt.copy_state", + "/update _vt.vreplication set state", + // The first fast-forward has no starting point. So, it just saves the current position. + "insert into dst1(id,val,val3,id2) values (1,'aaa','aaa1',10), (2,'bbb','bbb2',20)", + `/update _vt.copy_state set lastpk='fields: rows: ' where vrepl_id=.*`, + // copy of dst1 is done: delete from copy_state. + "/delete from _vt.copy_state.*dst1", + "insert into dst2(val3,val,id2) values ('aaa1','aaa',10), ('bbb2','bbb',20)", + `/update _vt.copy_state set lastpk='fields: rows: ' where vrepl_id=.*`, + // copy of dst2 is done: delete from copy_state. + "/delete from _vt.copy_state.*dst2", + "/update _vt.vreplication set state", + }) + expectData(t, "dst1", [][]string{ + {"1", "aaa", "1aaa", "aaa1", "10"}, + {"2", "bbb", "2bbb", "bbb2", "20"}, + }) + expectData(t, "dst2", [][]string{ + {"aaa1", "aaa", "10"}, + {"bbb2", "bbb", "20"}, + }) +} From 46f534726d7e39efd31c07dab2df673aca8db465 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Sat, 15 May 2021 11:33:47 +0200 Subject: [PATCH 23/64] Add vstreamer test to confirm generated columns are sent in events Signed-off-by: Rohit Nayak Signed-off-by: Rohit Nayak --- .../vreplication/vcopier_test.go | 2 +- .../tabletserver/vstreamer/vstreamer_test.go | 48 +++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/go/vt/vttablet/tabletmanager/vreplication/vcopier_test.go b/go/vt/vttablet/tabletmanager/vreplication/vcopier_test.go index d2f6198222..c6aed04523 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vcopier_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vcopier_test.go @@ -1285,7 +1285,7 @@ func TestPlayerCopyTablesWithGeneratedColumn(t *testing.T) { flavor := strings.ToLower(env.Flavor) // Disable tests on percona (which identifies as mysql56) and mariadb platforms in CI since they // generated columns support was added in 5.7 and mariadb added mysql compatible generated columns in 10.2 - if strings.Contains(flavor, "56") || strings.Contains(flavor, "maria") { + if !strings.Contains(flavor, "mysql57") && !strings.Contains(flavor, "mysql80") { return } defer deleteTablet(addTablet(100)) diff --git a/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go b/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go index e30f4b8271..0fbff97557 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go @@ -1922,6 +1922,54 @@ func TestFilteredMultipleWhere(t *testing.T) { runCases(t, filter, testcases, "", nil) } +// TestGeneratedColumns just confirms that generated columns are sent in a vstream as expected +func TestGeneratedColumns(t *testing.T) { + flavor := strings.ToLower(env.Flavor) + // Disable tests on percona (which identifies as mysql56) and mariadb platforms in CI since they + // generated columns support was added in 5.7 and mariadb added mysql compatible generated columns in 10.2 + if !strings.Contains(flavor, "mysql57") && !strings.Contains(flavor, "mysql80") { + return + } + execStatements(t, []string{ + "create table t1(id int, val varbinary(6), val2 varbinary(6) as (concat(id, val)), val3 varbinary(6) as (concat(val, id)), id2 int, primary key(id))", + }) + defer execStatements(t, []string{ + "drop table t1", + }) + engine.se.Reload(context.Background()) + queries := []string{ + "begin", + "insert into t1(id, val, id2) values (1, 'aaa', 10)", + "insert into t1(id, val, id2) values (2, 'bbb', 20)", + "commit", + } + + fe := &TestFieldEvent{ + table: "t1", + db: "vttest", + cols: []*TestColumn{ + {name: "id", dataType: "INT32", colType: "", len: 11, charset: 63}, + {name: "val", dataType: "VARBINARY", colType: "", len: 6, charset: 63}, + {name: "val2", dataType: "VARBINARY", colType: "", len: 6, charset: 63}, + {name: "val3", dataType: "VARBINARY", colType: "", len: 6, charset: 63}, + {name: "id2", dataType: "INT32", colType: "", len: 11, charset: 63}, + }, + } + + testcases := []testcase{{ + input: queries, + output: [][]string{{ + `begin`, + fe.String(), + `type:ROW row_event: > > `, + `type:ROW row_event: > > `, + `gtid`, + `commit`, + }}, + }} + runCases(t, nil, testcases, "current", nil) +} + func runCases(t *testing.T, filter *binlogdatapb.Filter, testcases []testcase, position string, tablePK []*binlogdatapb.TableLastPK) { ctx, cancel := context.WithCancel(context.Background()) From 38de519945251360be5d818278e520277cd89411 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Sun, 16 May 2021 17:09:05 +0200 Subject: [PATCH 24/64] Add vplayer test for generated columns Signed-off-by: Rohit Nayak --- .../vreplication/vplayer_flaky_test.go | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go b/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go index f5aaaea022..630310a3a8 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go @@ -2618,6 +2618,89 @@ func TestVReplicationLogs(t *testing.T) { } } +func TestGeneratedColumns(t *testing.T) { + defer deleteTablet(addTablet(100)) + + execStatements(t, []string{ + "create table t1(id int, val varbinary(6), val2 varbinary(6) as (concat(id, val)), val3 varbinary(6) as (concat(val, id)), id2 int, primary key(id))", + fmt.Sprintf("create table %s.t1(id int, val varbinary(6), val2 varbinary(6) as (concat(id, val)), val3 varbinary(6), id2 int, primary key(id))", vrepldb), + "create table t2(id int, val varbinary(128), val2 varbinary(128) as (concat(id, val)) stored, val3 varbinary(128) as (concat(val, id)), id2 int, primary key(id))", + fmt.Sprintf("create table %s.t2(id int, val3 varbinary(128), val varbinary(128), id2 int, primary key(id))", vrepldb), + }) + defer execStatements(t, []string{ + "drop table t1", + fmt.Sprintf("drop table %s.t1", vrepldb), + "drop table t2", + fmt.Sprintf("drop table %s.t2", vrepldb), + }) + env.SchemaEngine.Reload(context.Background()) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + Filter: "select * from t1", + }, { + Match: "t2", + Filter: "select id, val3, val, id2 from t2", + }}, + } + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + cancel, _ := startVReplication(t, bls, "") + defer cancel() + + testcases := []struct { + input string + output string + table string + data [][]string + }{{ + input: "insert into t1(id, val, id2) values (1, 'aaa', 10)", + output: "insert into t1(id,val,val3,id2) values (1,'aaa','aaa1',10)", + table: "t1", + data: [][]string{ + {"1", "aaa", "1aaa", "aaa1", "10"}, + }, + }, { + input: "update t1 set val = 'bbb', id2 = 11 where id = 1", + output: "update t1 set val='bbb', val3='bbb1', id2=11 where id=1", + table: "t1", + data: [][]string{ + {"1", "bbb", "1bbb", "bbb1", "11"}, + }, + }, { + input: "insert into t2(id, val, id2) values (1, 'aaa', 10)", + output: "insert into t2(id,val3,val,id2) values (1,'aaa1','aaa',10)", + table: "t2", + data: [][]string{ + {"1", "aaa1", "aaa", "10"}, + }, + }, { + input: "update t2 set val = 'bbb', id2 = 11 where id = 1", + output: "update t2 set val3='bbb1', val='bbb', id2=11 where id=1", + table: "t2", + data: [][]string{ + {"1", "bbb1", "bbb", "11"}, + }, + }} + + for _, tcases := range testcases { + execStatements(t, []string{tcases.input}) + output := []string{ + tcases.output, + } + expectNontxQueries(t, output) + time.Sleep(1 * time.Second) + if tcases.table != "" { + expectData(t, tcases.table, tcases.data) + } + } +} + func expectJSON(t *testing.T, table string, values [][]string, id int, exec func(ctx context.Context, query string) (*sqltypes.Result, error)) { t.Helper() From 7789c30528deac4371a8a393d16d03c2d4f51c3a Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Sun, 16 May 2021 17:30:33 +0200 Subject: [PATCH 25/64] Add vplayer test for generated columns Signed-off-by: Rohit Nayak --- .../tabletmanager/vreplication/vplayer_flaky_test.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go b/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go index 630310a3a8..bf4d20a98f 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go @@ -2619,6 +2619,12 @@ func TestVReplicationLogs(t *testing.T) { } func TestGeneratedColumns(t *testing.T) { + flavor := strings.ToLower(env.Flavor) + // Disable tests on percona (which identifies as mysql56) and mariadb platforms in CI since they + // generated columns support was added in 5.7 and mariadb added mysql compatible generated columns in 10.2 + if !strings.Contains(flavor, "mysql57") && !strings.Contains(flavor, "mysql80") { + return + } defer deleteTablet(addTablet(100)) execStatements(t, []string{ @@ -2694,7 +2700,6 @@ func TestGeneratedColumns(t *testing.T) { tcases.output, } expectNontxQueries(t, output) - time.Sleep(1 * time.Second) if tcases.table != "" { expectData(t, tcases.table, tcases.data) } From 6d9e3cbe5eea10bb4f361f8a746d1de3bbdfb6e8 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Mon, 7 Jun 2021 11:16:07 +0200 Subject: [PATCH 26/64] Improve comment Signed-off-by: Rohit Nayak --- go/vt/sqlparser/parsed_query.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/sqlparser/parsed_query.go b/go/vt/sqlparser/parsed_query.go index 7651c4bc48..1fff8b37c3 100644 --- a/go/vt/sqlparser/parsed_query.go +++ b/go/vt/sqlparser/parsed_query.go @@ -89,7 +89,7 @@ func (pq *ParsedQuery) Append(buf *strings.Builder, bindVariables map[string]*qu // the fields in the row are in the same order as the placeholders in this query. The fields might include generated // columns which are dropped, by checking against skipFields, before binding the variables // note: there can be more fields than bind locations since extra columns might be requested from the source if not all -// primary keys columns are not in the select list, for example. Also some values in the row may not correspond for +// primary keys columns are present in the target table, for example. Also some values in the row may not correspond for // values from the database on the source: sum/count for aggregation queries, for example func (pq *ParsedQuery) AppendFromRow(buf *bytes2.Buffer, fields []*querypb.Field, row *querypb.Row, skipFields map[string]bool) error { if len(fields) < len(pq.bindLocations) { From 97e34650cd1429a178aa20bae2c7097ce60bc03d Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Mon, 7 Jun 2021 17:58:46 +0530 Subject: [PATCH 27/64] use tableschemachanges from the stats sent from tablet to vtgate Signed-off-by: Harshit Gangal --- go/mysql/schema.go | 2 +- go/test/endtoend/vtgate/main_test.go | 2 +- go/test/endtoend/vtgate/misc_test.go | 43 ++++++++++++++++--- go/vt/vtgate/schema/tracker.go | 7 +-- go/vt/vtgate/schema/tracker_test.go | 10 ++--- go/vt/vtgate/schema/uptate_controller.go | 10 ++--- go/vt/vtgate/schema/uptate_controller_test.go | 10 ++--- go/vt/vtgate/vschema_manager.go | 3 ++ 8 files changed, 62 insertions(+), 25 deletions(-) diff --git a/go/mysql/schema.go b/go/mysql/schema.go index 099d102204..59f98f0dbf 100644 --- a/go/mysql/schema.go +++ b/go/mysql/schema.go @@ -99,7 +99,7 @@ where table_schema = database()` FetchUpdatedTables = `select table_name, column_name, data_type from _vt.schemacopy where table_schema = database() and - table_name in :tableNames + table_name in ::tableNames order by table_name, ordinal_position` // FetchTables queries fetches all information about tables diff --git a/go/test/endtoend/vtgate/main_test.go b/go/test/endtoend/vtgate/main_test.go index 7fda9e8fbe..a8b8baea1d 100644 --- a/go/test/endtoend/vtgate/main_test.go +++ b/go/test/endtoend/vtgate/main_test.go @@ -418,7 +418,7 @@ func TestMain(m *testing.M) { VSchema: VSchema, } clusterInstance.VtGateExtraArgs = []string{"-schema_change_signal"} - clusterInstance.VtTabletExtraArgs = []string{"-queryserver-config-schema-change-signal"} + clusterInstance.VtTabletExtraArgs = []string{"-queryserver-config-schema-change-signal", "-queryserver-config-schema-change-signal-interval", "0.1"} err = clusterInstance.StartKeyspace(*keyspace, []string{"-80", "80-"}, 1, true) if err != nil { return 1 diff --git a/go/test/endtoend/vtgate/misc_test.go b/go/test/endtoend/vtgate/misc_test.go index 77de7f4076..2a29f5ef3d 100644 --- a/go/test/endtoend/vtgate/misc_test.go +++ b/go/test/endtoend/vtgate/misc_test.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "testing" + "time" "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/assert" @@ -662,16 +663,31 @@ func TestSchemaTracker(t *testing.T) { require.NoError(t, err) } -func TestVSchemaTrackerInit(t *testing.T) { +func TestVSchemaTrackerInitAndUpdate(t *testing.T) { ctx := context.Background() conn, err := mysql.Connect(ctx, &vtParams) require.NoError(t, err) defer conn.Close() - qr := exec(t, conn, "SHOW VSCHEMA TABLES") - got := fmt.Sprintf("%v", qr.Rows) - want := `[[VARCHAR("aggr_test")] [VARCHAR("dual")] [VARCHAR("t1")] [VARCHAR("t1_id2_idx")] [VARCHAR("t2")] [VARCHAR("t2_id4_idx")] [VARCHAR("t3")] [VARCHAR("t3_id7_idx")] [VARCHAR("t4")] [VARCHAR("t4_id2_idx")] [VARCHAR("t5_null_vindex")] [VARCHAR("t6")] [VARCHAR("t6_id2_idx")] [VARCHAR("t7_fk")] [VARCHAR("t7_xxhash")] [VARCHAR("t7_xxhash_idx")] [VARCHAR("t8")] [VARCHAR("vstream_test")]]` - assert.Equal(t, want, got) + assertMatches(t, conn, "SHOW VSCHEMA TABLES", `[[VARCHAR("aggr_test")] [VARCHAR("dual")] [VARCHAR("t1")] [VARCHAR("t1_id2_idx")] [VARCHAR("t2")] [VARCHAR("t2_id4_idx")] [VARCHAR("t3")] [VARCHAR("t3_id7_idx")] [VARCHAR("t4")] [VARCHAR("t4_id2_idx")] [VARCHAR("t5_null_vindex")] [VARCHAR("t6")] [VARCHAR("t6_id2_idx")] [VARCHAR("t7_fk")] [VARCHAR("t7_xxhash")] [VARCHAR("t7_xxhash_idx")] [VARCHAR("t8")] [VARCHAR("vstream_test")]]`) + + // Init + _ = exec(t, conn, "create table test_sc (id bigint primary key)") + assertMatchesWithTimeout(t, conn, + "SHOW VSCHEMA TABLES", + `[[VARCHAR("aggr_test")] [VARCHAR("dual")] [VARCHAR("t1")] [VARCHAR("t1_id2_idx")] [VARCHAR("t2")] [VARCHAR("t2_id4_idx")] [VARCHAR("t3")] [VARCHAR("t3_id7_idx")] [VARCHAR("t4")] [VARCHAR("t4_id2_idx")] [VARCHAR("t5_null_vindex")] [VARCHAR("t6")] [VARCHAR("t6_id2_idx")] [VARCHAR("t7_fk")] [VARCHAR("t7_xxhash")] [VARCHAR("t7_xxhash_idx")] [VARCHAR("t8")] [VARCHAR("test_sc")] [VARCHAR("vstream_test")]]`, + 100*time.Millisecond, + 3*time.Second, + "test_sc not in vschema tables") + + // Tables Update via health check. + _ = exec(t, conn, "create table test_sc1 (id bigint primary key)") + assertMatchesWithTimeout(t, conn, + "SHOW VSCHEMA TABLES", + `[[VARCHAR("aggr_test")] [VARCHAR("dual")] [VARCHAR("t1")] [VARCHAR("t1_id2_idx")] [VARCHAR("t2")] [VARCHAR("t2_id4_idx")] [VARCHAR("t3")] [VARCHAR("t3_id7_idx")] [VARCHAR("t4")] [VARCHAR("t4_id2_idx")] [VARCHAR("t5_null_vindex")] [VARCHAR("t6")] [VARCHAR("t6_id2_idx")] [VARCHAR("t7_fk")] [VARCHAR("t7_xxhash")] [VARCHAR("t7_xxhash_idx")] [VARCHAR("t8")] [VARCHAR("test_sc")] [VARCHAR("test_sc1")] [VARCHAR("vstream_test")]]`, + 100*time.Millisecond, + 3*time.Second, + "test_sc1 not in vschema tables") } func assertMatches(t *testing.T, conn *mysql.Conn, query, expected string) { @@ -683,6 +699,23 @@ func assertMatches(t *testing.T, conn *mysql.Conn, query, expected string) { t.Errorf("Query: %s (-want +got):\n%s", query, diff) } } + +func assertMatchesWithTimeout(t *testing.T, conn *mysql.Conn, query, expected string, r time.Duration, d time.Duration, failureMsg string) { + t.Helper() + timeout := time.After(d) + diff := "actual and expectation does not match" + for len(diff) > 0 { + select { + case <-timeout: + require.Fail(t, failureMsg, diff) + case <-time.After(r): + qr := exec(t, conn, `SHOW VSCHEMA TABLES`) + diff = cmp.Diff(expected, + fmt.Sprintf("%v", qr.Rows)) + } + + } +} func assertMatchesNoOrder(t *testing.T, conn *mysql.Conn, query, expected string) { t.Helper() qr := exec(t, conn, query) diff --git a/go/vt/vtgate/schema/tracker.go b/go/vt/vtgate/schema/tracker.go index 824ada235b..3fe331dd1b 100644 --- a/go/vt/vtgate/schema/tracker.go +++ b/go/vt/vtgate/schema/tracker.go @@ -149,7 +149,8 @@ func (t *Tracker) Tables(ks string) map[string][]vindexes.Column { } func (t *Tracker) updateSchema(th *discovery.TabletHealth) bool { - tables, err := sqltypes.BuildBindVariable(th.TablesUpdated) + tablesUpdated := th.Stats.TableSchemaChanged + tables, err := sqltypes.BuildBindVariable(tablesUpdated) if err != nil { log.Errorf("failed to read updated tables from TabletHealth: %v", err) return false @@ -158,7 +159,7 @@ func (t *Tracker) updateSchema(th *discovery.TabletHealth) bool { res, err := th.Conn.Execute(t.ctx, th.Target, mysql.FetchUpdatedTables, bv, 0, 0, nil) if err != nil { // TODO: these tables should now become non-authoritative - log.Warningf("error fetching new schema for %v, making them non-authoritative: %v", th.TablesUpdated, err) + log.Warningf("error fetching new schema for %v, making them non-authoritative: %v", tablesUpdated, err) return false } @@ -167,7 +168,7 @@ func (t *Tracker) updateSchema(th *discovery.TabletHealth) bool { // first we empty all prior schema. deleted tables will not show up in the result, // so this is the only chance to delete - for _, tbl := range th.TablesUpdated { + for _, tbl := range tablesUpdated { t.tables.delete(th.Target.Keyspace, tbl) } t.updateTables(th.Target.Keyspace, res) diff --git a/go/vt/vtgate/schema/tracker_test.go b/go/vt/vtgate/schema/tracker_test.go index b1fd591f75..53110df668 100644 --- a/go/vt/vtgate/schema/tracker_test.go +++ b/go/vt/vtgate/schema/tracker_test.go @@ -168,11 +168,11 @@ func TestTracking(t *testing.T) { for _, d := range tcase.deltas { ch <- &discovery.TabletHealth{ - Conn: sbc, - Tablet: tablet, - Target: target, - Serving: true, - TablesUpdated: d.updTbl, + Conn: sbc, + Tablet: tablet, + Target: target, + Serving: true, + Stats: &querypb.RealtimeStats{TableSchemaChanged: d.updTbl}, } } diff --git a/go/vt/vtgate/schema/uptate_controller.go b/go/vt/vtgate/schema/uptate_controller.go index 4a099bf89b..da22967627 100644 --- a/go/vt/vtgate/schema/uptate_controller.go +++ b/go/vt/vtgate/schema/uptate_controller.go @@ -70,18 +70,18 @@ func (u *updateController) consume() { func (u *updateController) getItemFromQueueLocked() *discovery.TabletHealth { item := u.queue.items[0] - i := 0 + i := 1 for ; i < len(u.queue.items); i++ { - for _, table := range u.queue.items[i].TablesUpdated { + for _, table := range u.queue.items[i].Stats.TableSchemaChanged { found := false - for _, itemTable := range item.TablesUpdated { + for _, itemTable := range item.Stats.TableSchemaChanged { if itemTable == table { found = true break } } if !found { - item.TablesUpdated = append(item.TablesUpdated, table) + item.Stats.TableSchemaChanged = append(item.Stats.TableSchemaChanged, table) } } } @@ -94,7 +94,7 @@ func (u *updateController) add(th *discovery.TabletHealth) { u.mu.Lock() defer u.mu.Unlock() - if len(th.TablesUpdated) == 0 && u.init == nil { + if len(th.Stats.TableSchemaChanged) == 0 && u.init == nil { return } if u.queue == nil { diff --git a/go/vt/vtgate/schema/uptate_controller_test.go b/go/vt/vtgate/schema/uptate_controller_test.go index 8d0c8ca501..db745957ef 100644 --- a/go/vt/vtgate/schema/uptate_controller_test.go +++ b/go/vt/vtgate/schema/uptate_controller_test.go @@ -118,7 +118,7 @@ func TestMultipleUpdatesFromDifferentShards(t *testing.T) { var signalNb, initNb int var updatedTables []string update := func(th *discovery.TabletHealth) bool { - updatedTables = th.TablesUpdated + updatedTables = th.Stats.TableSchemaChanged return !test.updateFail } signal := func() { @@ -148,10 +148,10 @@ func TestMultipleUpdatesFromDifferentShards(t *testing.T) { Type: target.TabletType, } d := &discovery.TabletHealth{ - Tablet: tablet, - Target: target, - Serving: true, - TablesUpdated: in.tablesUpdates, + Tablet: tablet, + Target: target, + Serving: true, + Stats: &querypb.RealtimeStats{TableSchemaChanged: in.tablesUpdates}, } if test.delay > 0 { time.Sleep(test.delay) diff --git a/go/vt/vtgate/vschema_manager.go b/go/vt/vtgate/vschema_manager.go index cc10197840..ccc7b8ae66 100644 --- a/go/vt/vtgate/vschema_manager.go +++ b/go/vt/vtgate/vschema_manager.go @@ -154,7 +154,9 @@ func (vm *VSchemaManager) Rebuild() { v := vm.currentSrvVschema vm.mu.Unlock() + log.Infof("Received schema update") if v == nil { + log.Infof("No vschema to enhance") return } @@ -165,6 +167,7 @@ func (vm *VSchemaManager) Rebuild() { if vm.subscriber != nil { vm.subscriber(vschema, vSchemaStats(nil, vschema)) + log.Infof("Sent vschema to subscriber") } } From d0714c7721e76cf021cf23a9b45d480079b8875e Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Mon, 7 Jun 2021 14:40:37 +0200 Subject: [PATCH 28/64] Fix merge Signed-off-by: Rohit Nayak --- go/test/endtoend/vreplication/helper.go | 4 ++-- go/test/endtoend/vreplication/vreplication_test_env.go | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go/test/endtoend/vreplication/helper.go b/go/test/endtoend/vreplication/helper.go index 15726e2dd3..01a80b2798 100644 --- a/go/test/endtoend/vreplication/helper.go +++ b/go/test/endtoend/vreplication/helper.go @@ -189,11 +189,11 @@ func validateDryRunResults(t *testing.T, output string, want []string) { } if !match { fail = true - t.Logf("want %s, got %s\n", w, gotDryRun[i]) + t.Fatalf("want %s, got %s\n", w, gotDryRun[i]) } } if fail { - t.Fatal("Dry run results don't match") + t.Fatalf("Dry run results don't match, want %s, got %s", want, gotDryRun) } } diff --git a/go/test/endtoend/vreplication/vreplication_test_env.go b/go/test/endtoend/vreplication/vreplication_test_env.go index 240649ce7b..244025b83d 100644 --- a/go/test/endtoend/vreplication/vreplication_test_env.go +++ b/go/test/endtoend/vreplication/vreplication_test_env.go @@ -49,10 +49,10 @@ var dryRunResultsReadCustomerShard = []string{ var dryRunResultsSwitchWritesM2m3 = []string{ "Lock keyspace merchant", "Stop streams on keyspace merchant", - "/ Id 2 Keyspace customer Shard -80 Rules rules:{match:\"morders\" filter:\"select * from orders where in_keyrange(mname, 'merchant.md5', '-80')\"} at Position ", - "/ Id 2 Keyspace customer Shard -80 Rules rules:{match:\"morders\" filter:\"select * from orders where in_keyrange(mname, 'merchant.md5', '80-')\"} at Position ", - "/ Id 3 Keyspace customer Shard 80- Rules rules:{match:\"morders\" filter:\"select * from orders where in_keyrange(mname, 'merchant.md5', '-80')\"} at Position ", - "/ Id 3 Keyspace customer Shard 80- Rules rules:{match:\"morders\" filter:\"select * from orders where in_keyrange(mname, 'merchant.md5', '80-')\"} at Position ", + "/ Id 2 Keyspace customer Shard -80 Rules rules:{match:\"morders\" filter:\"select oid, cid, mname, pid, price, qty, total from orders where in_keyrange(mname, 'merchant.md5', '-80')\"} at Position ", + "/ Id 2 Keyspace customer Shard -80 Rules rules:{match:\"morders\" filter:\"select oid, cid, mname, pid, price, qty, total from orders where in_keyrange(mname, 'merchant.md5', '80-')\"} at Position ", + "/ Id 3 Keyspace customer Shard 80- Rules rules:{match:\"morders\" filter:\"select oid, cid, mname, pid, price, qty, total from orders where in_keyrange(mname, 'merchant.md5', '-80')\"} at Position ", + "/ Id 3 Keyspace customer Shard 80- Rules rules:{match:\"morders\" filter:\"select oid, cid, mname, pid, price, qty, total from orders where in_keyrange(mname, 'merchant.md5', '80-')\"} at Position ", "/ Id 4 Keyspace customer Shard -80 Rules rules:{match:\"msales\" filter:\"select mname as merchant_name, count(*) as kount, sum(price) as amount from orders where in_keyrange(mname, 'merchant.md5', '-80') group by merchant_name\"} at Position ", "/ Id 4 Keyspace customer Shard -80 Rules rules:{match:\"msales\" filter:\"select mname as merchant_name, count(*) as kount, sum(price) as amount from orders where in_keyrange(mname, 'merchant.md5', '80-') group by merchant_name\"} at Position ", "/ Id 5 Keyspace customer Shard 80- Rules rules:{match:\"msales\" filter:\"select mname as merchant_name, count(*) as kount, sum(price) as amount from orders where in_keyrange(mname, 'merchant.md5', '-80') group by merchant_name\"} at Position ", From 3ff1a32ca3a4a75822ffaf4d497d95f66bc39318 Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Mon, 7 Jun 2021 19:01:00 +0530 Subject: [PATCH 29/64] add reload Keyspace signal to update controller to load full schema for the keyspace when it is set and set to false when successfully loaded Signed-off-by: Harshit Gangal --- go/vt/vtgate/schema/tracker.go | 48 ++++++++++++------- go/vt/vtgate/schema/tracker_test.go | 8 ++-- go/vt/vtgate/schema/uptate_controller.go | 24 +++++----- go/vt/vtgate/schema/uptate_controller_test.go | 8 ++-- go/vt/vtgate/vtgate.go | 2 +- 5 files changed, 49 insertions(+), 41 deletions(-) diff --git a/go/vt/vtgate/schema/tracker.go b/go/vt/vtgate/schema/tracker.go index 3fe331dd1b..6f43590983 100644 --- a/go/vt/vtgate/schema/tracker.go +++ b/go/vt/vtgate/schema/tracker.go @@ -34,8 +34,8 @@ import ( ) type ( - keyspace = string - tableName = string + keyspaceStr = string + tableNameStr = string // Tracker contains the required fields to perform schema tracking. Tracker struct { @@ -48,7 +48,7 @@ type ( signal func() // a function that we'll call whenever we have new schema data // map of keyspace currently tracked - tracked map[keyspace]*updateController + tracked map[keyspaceStr]*updateController consumeDelay time.Duration } ) @@ -61,8 +61,8 @@ func NewTracker(ch chan *discovery.TabletHealth) *Tracker { return &Tracker{ ctx: context.Background(), ch: ch, - tables: &tableMap{m: map[keyspace]map[tableName][]vindexes.Column{}}, - tracked: map[keyspace]*updateController{}, + tables: &tableMap{m: map[keyspaceStr]map[tableNameStr][]vindexes.Column{}}, + tracked: map[keyspaceStr]*updateController{}, consumeDelay: defaultConsumeDelay, } } @@ -76,6 +76,7 @@ func (t *Tracker) LoadKeyspace(conn queryservice.QueryService, target *querypb.T t.mu.Lock() defer t.mu.Unlock() t.updateTables(target.Keyspace, res) + t.tracked[target.Keyspace].loaded = true log.Infof("finished loading schema for keyspace %s. Found %d tables", target.Keyspace, len(res.Rows)) return nil } @@ -105,22 +106,27 @@ func (t *Tracker) getKeyspaceUpdateController(th *discovery.TabletHealth) *updat t.mu.Lock() defer t.mu.Unlock() - ksUpdater, ok := t.tracked[th.Target.Keyspace] - if !ok { - init := func(th *discovery.TabletHealth) bool { - err := t.LoadKeyspace(th.Conn, th.Target) - if err != nil { - log.Warningf("Unable to add keyspace to tracker: %v", err) - return false - } - return true - } - ksUpdater = &updateController{update: t.updateSchema, init: init, signal: t.signal, consumeDelay: t.consumeDelay} + ksUpdater, exists := t.tracked[th.Target.Keyspace] + if !exists { + ksUpdater = t.newUpdateController() t.tracked[th.Target.Keyspace] = ksUpdater } return ksUpdater } +func (t *Tracker) newUpdateController() *updateController { + return &updateController{update: t.updateSchema, reloadKeyspace: t.initKeyspace, signal: t.signal, consumeDelay: t.consumeDelay} +} + +func (t *Tracker) initKeyspace(th *discovery.TabletHealth) bool { + err := t.LoadKeyspace(th.Conn, th.Target) + if err != nil { + log.Warningf("Unable to add keyspace to tracker: %v", err) + return false + } + return true +} + // Stop stops the schema tracking func (t *Tracker) Stop() { log.Info("Stopping schema tracking") @@ -194,14 +200,20 @@ func (t *Tracker) RegisterSignalReceiver(f func()) { t.signal = f } +// AddNewKeyspace adds keyspace to the tracker. +func (t *Tracker) AddNewKeyspace(conn queryservice.QueryService, target *querypb.Target) error { + t.tracked[target.Keyspace] = t.newUpdateController() + return t.LoadKeyspace(conn, target) +} + type tableMap struct { - m map[keyspace]map[tableName][]vindexes.Column + m map[keyspaceStr]map[tableNameStr][]vindexes.Column } func (tm *tableMap) set(ks, tbl string, cols []vindexes.Column) { m := tm.m[ks] if m == nil { - m = make(map[tableName][]vindexes.Column) + m = make(map[tableNameStr][]vindexes.Column) tm.m[ks] = m } m[tbl] = cols diff --git a/go/vt/vtgate/schema/tracker_test.go b/go/vt/vtgate/schema/tracker_test.go index 53110df668..7789fa05bd 100644 --- a/go/vt/vtgate/schema/tracker_test.go +++ b/go/vt/vtgate/schema/tracker_test.go @@ -207,7 +207,7 @@ func waitTimeout(wg *sync.WaitGroup, timeout time.Duration) bool { func TestTrackerGetKeyspaceUpdateController(t *testing.T) { ks3 := &updateController{} tracker := Tracker{ - tracked: map[keyspace]*updateController{ + tracked: map[keyspaceStr]*updateController{ "ks3": ks3, }, } @@ -231,7 +231,7 @@ func TestTrackerGetKeyspaceUpdateController(t *testing.T) { assert.Equal(t, ks2, tracker.getKeyspaceUpdateController(th2), "received different updateController") assert.Equal(t, ks3, tracker.getKeyspaceUpdateController(th3), "received different updateController") - assert.NotNil(t, ks1.init, "ks1 needs to be initialized") - assert.NotNil(t, ks2.init, "ks2 needs to be initialized") - assert.Nil(t, ks3.init, "ks3 already initialized") + assert.NotNil(t, ks1.reloadKeyspace, "ks1 needs to be initialized") + assert.NotNil(t, ks2.reloadKeyspace, "ks2 needs to be initialized") + assert.Nil(t, ks3.reloadKeyspace, "ks3 already initialized") } diff --git a/go/vt/vtgate/schema/uptate_controller.go b/go/vt/vtgate/schema/uptate_controller.go index da22967627..892bdc9af5 100644 --- a/go/vt/vtgate/schema/uptate_controller.go +++ b/go/vt/vtgate/schema/uptate_controller.go @@ -29,12 +29,13 @@ type ( } updateController struct { - mu sync.Mutex - queue *queue - consumeDelay time.Duration - update func(th *discovery.TabletHealth) bool - init func(th *discovery.TabletHealth) bool - signal func() + mu sync.Mutex + queue *queue + consumeDelay time.Duration + update func(th *discovery.TabletHealth) bool + reloadKeyspace func(th *discovery.TabletHealth) bool + signal func() + loaded bool } ) @@ -54,13 +55,10 @@ func (u *updateController) consume() { u.mu.Unlock() var success bool - if u.init != nil { - success = u.init(item) - if success { - u.init = nil - } - } else { + if u.loaded { success = u.update(item) + } else { + success = u.reloadKeyspace(item) } if success && u.signal != nil { u.signal() @@ -94,7 +92,7 @@ func (u *updateController) add(th *discovery.TabletHealth) { u.mu.Lock() defer u.mu.Unlock() - if len(th.Stats.TableSchemaChanged) == 0 && u.init == nil { + if len(th.Stats.TableSchemaChanged) == 0 && u.loaded { return } if u.queue == nil { diff --git a/go/vt/vtgate/schema/uptate_controller_test.go b/go/vt/vtgate/schema/uptate_controller_test.go index db745957ef..04895fccdf 100644 --- a/go/vt/vtgate/schema/uptate_controller_test.go +++ b/go/vt/vtgate/schema/uptate_controller_test.go @@ -128,13 +128,11 @@ func TestMultipleUpdatesFromDifferentShards(t *testing.T) { update: update, signal: signal, consumeDelay: 5 * time.Millisecond, - } - - if test.init { - kUpdate.init = func(th *discovery.TabletHealth) bool { + reloadKeyspace: func(th *discovery.TabletHealth) bool { initNb++ return !test.initFail - } + }, + loaded: !test.init, } for _, in := range test.inputs { diff --git a/go/vt/vtgate/vtgate.go b/go/vt/vtgate/vtgate.go index e098882174..f751d7fc85 100644 --- a/go/vt/vtgate/vtgate.go +++ b/go/vt/vtgate/vtgate.go @@ -311,7 +311,7 @@ func resolveAndLoadKeyspace(ctx context.Context, srvResolver *srvtopo.Resolver, return case <-time.After(500 * time.Millisecond): for _, shard := range dest { - err := st.LoadKeyspace(gw, shard.Target) + err := st.AddNewKeyspace(gw, shard.Target) if err == nil { return } From 94f44eeaa8664222b784b40e024503bd42dda64d Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Mon, 7 Jun 2021 20:05:33 +0530 Subject: [PATCH 30/64] for non-serving primary heathchech update, mark the tracker to reload the full schema next time. if the updates tables fetches error, mark the tracker to reload the full schema next time. Signed-off-by: Harshit Gangal --- go/vt/discovery/tablet_health.go | 3 - go/vt/vtgate/schema/tracker.go | 3 +- go/vt/vtgate/schema/tracker_test.go | 70 ++++++++++++++++++++++++ go/vt/vtgate/schema/uptate_controller.go | 9 +++ 4 files changed, 81 insertions(+), 4 deletions(-) diff --git a/go/vt/discovery/tablet_health.go b/go/vt/discovery/tablet_health.go index 02b8b1c27b..1a28639348 100644 --- a/go/vt/discovery/tablet_health.go +++ b/go/vt/discovery/tablet_health.go @@ -39,9 +39,6 @@ type TabletHealth struct { MasterTermStartTime int64 LastError error Serving bool - - // TablesUpdated contains a list of all tables that we need to fetch new schema info for - TablesUpdated []string } // DeepEqual compares two TabletHealth. Since we include protos, we diff --git a/go/vt/vtgate/schema/tracker.go b/go/vt/vtgate/schema/tracker.go index 6f43590983..638c2a428d 100644 --- a/go/vt/vtgate/schema/tracker.go +++ b/go/vt/vtgate/schema/tracker.go @@ -164,7 +164,8 @@ func (t *Tracker) updateSchema(th *discovery.TabletHealth) bool { bv := map[string]*querypb.BindVariable{"tableNames": tables} res, err := th.Conn.Execute(t.ctx, th.Target, mysql.FetchUpdatedTables, bv, 0, 0, nil) if err != nil { - // TODO: these tables should now become non-authoritative + t.tracked[th.Target.Keyspace].loaded = false + // TODO: optimize for the tables that got errored out. log.Warningf("error fetching new schema for %v, making them non-authoritative: %v", tablesUpdated, err) return false } diff --git a/go/vt/vtgate/schema/tracker_test.go b/go/vt/vtgate/schema/tracker_test.go index 7789fa05bd..4504bac689 100644 --- a/go/vt/vtgate/schema/tracker_test.go +++ b/go/vt/vtgate/schema/tracker_test.go @@ -22,6 +22,8 @@ import ( "testing" "time" + "vitess.io/vitess/go/mysql" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -190,6 +192,74 @@ func TestTracking(t *testing.T) { } } +func TestTrackingUnHealthyTablet(t *testing.T) { + target := &querypb.Target{ + Keyspace: "ks", + Shard: "-80", + TabletType: topodatapb.TabletType_MASTER, + Cell: "aa", + } + tablet := &topodatapb.Tablet{ + Keyspace: target.Keyspace, + Shard: target.Shard, + Type: target.TabletType, + } + + sbc := sandboxconn.NewSandboxConn(tablet) + ch := make(chan *discovery.TabletHealth) + tracker := NewTracker(ch) + tracker.consumeDelay = 1 * time.Millisecond + tracker.Start() + defer tracker.Stop() + + // the test are written in a way that it expects 3 signals to be sent from the tracker to the subscriber. + wg := sync.WaitGroup{} + wg.Add(3) + tracker.RegisterSignalReceiver(func() { + wg.Done() + }) + + tcases := []struct { + name string + serving bool + expectedQuery string + updatedTbls []string + }{ + { + name: "initial load", + serving: true, + }, + { + name: "initial load", + serving: true, + updatedTbls: []string{"a"}, + }, + { + name: "non serving tablet", + serving: false, + }, + { + name: "now serving tablet", + serving: true, + }, + } + + sbc.SetResults([]*sqltypes.Result{{}, {}, {}}) + for _, tcase := range tcases { + ch <- &discovery.TabletHealth{ + Conn: sbc, + Tablet: tablet, + Target: target, + Serving: tcase.serving, + Stats: &querypb.RealtimeStats{TableSchemaChanged: tcase.updatedTbls}, + } + time.Sleep(5 * time.Millisecond) + } + + require.False(t, waitTimeout(&wg, time.Second), "schema was updated but received no signal") + require.Equal(t, []string{mysql.FetchTables, mysql.FetchUpdatedTables, mysql.FetchTables}, sbc.StringQueries()) +} + func waitTimeout(wg *sync.WaitGroup, timeout time.Duration) bool { c := make(chan struct{}) go func() { diff --git a/go/vt/vtgate/schema/uptate_controller.go b/go/vt/vtgate/schema/uptate_controller.go index 892bdc9af5..165a661397 100644 --- a/go/vt/vtgate/schema/uptate_controller.go +++ b/go/vt/vtgate/schema/uptate_controller.go @@ -20,6 +20,8 @@ import ( "sync" "time" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" + "vitess.io/vitess/go/vt/discovery" ) @@ -92,6 +94,13 @@ func (u *updateController) add(th *discovery.TabletHealth) { u.mu.Lock() defer u.mu.Unlock() + // Received a health check from primary tablet that is it not reachable from VTGate. + // The connection will get reset and the tracker needs to reload the schema for the keyspace. + if th.Tablet.Type == topodatapb.TabletType_MASTER && !th.Serving { + u.loaded = false + return + } + if len(th.Stats.TableSchemaChanged) == 0 && u.loaded { return } From 2c0283695ce618700f9056d29fd7d6c1afff7cc2 Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Mon, 7 Jun 2021 23:05:14 +0530 Subject: [PATCH 31/64] update signal on update controller when the signal function is registered with tracker Signed-off-by: Harshit Gangal --- go/vt/vtgate/schema/tracker.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/go/vt/vtgate/schema/tracker.go b/go/vt/vtgate/schema/tracker.go index 638c2a428d..081907f691 100644 --- a/go/vt/vtgate/schema/tracker.go +++ b/go/vt/vtgate/schema/tracker.go @@ -198,6 +198,11 @@ func (t *Tracker) updateTables(keyspace string, res *sqltypes.Result) { // RegisterSignalReceiver allows a function to register to be called when new schema is available func (t *Tracker) RegisterSignalReceiver(f func()) { + t.mu.Lock() + defer t.mu.Unlock() + for _, controller := range t.tracked { + controller.signal = f + } t.signal = f } From 828f3b88a31f555ab1e3817b52e739309d120b5f Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Mon, 7 Jun 2021 23:21:21 +0530 Subject: [PATCH 32/64] find uniq tables only when updated tables schema is to be fetched Signed-off-by: Harshit Gangal --- go/vt/vtgate/schema/uptate_controller.go | 27 +++++++++++++----------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/go/vt/vtgate/schema/uptate_controller.go b/go/vt/vtgate/schema/uptate_controller.go index 165a661397..7d6b3a7d7a 100644 --- a/go/vt/vtgate/schema/uptate_controller.go +++ b/go/vt/vtgate/schema/uptate_controller.go @@ -70,23 +70,26 @@ func (u *updateController) consume() { func (u *updateController) getItemFromQueueLocked() *discovery.TabletHealth { item := u.queue.items[0] - i := 1 - for ; i < len(u.queue.items); i++ { - for _, table := range u.queue.items[i].Stats.TableSchemaChanged { - found := false - for _, itemTable := range item.Stats.TableSchemaChanged { - if itemTable == table { - found = true - break + itemsCount := len(u.queue.items) + // Only when we want to update selected tables. + if u.loaded { + for i := 1; i < itemsCount; i++ { + for _, table := range u.queue.items[i].Stats.TableSchemaChanged { + found := false + for _, itemTable := range item.Stats.TableSchemaChanged { + if itemTable == table { + found = true + break + } + } + if !found { + item.Stats.TableSchemaChanged = append(item.Stats.TableSchemaChanged, table) } - } - if !found { - item.Stats.TableSchemaChanged = append(item.Stats.TableSchemaChanged, table) } } } // emptying queue's items as all items from 0 to i (length of the queue) are merged - u.queue.items = u.queue.items[i:] + u.queue.items = u.queue.items[itemsCount:] return item } From 902ebf2a01dc10cf4cb5a67ace8f8c52217a4cab Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Tue, 18 May 2021 11:34:06 +0200 Subject: [PATCH 33/64] If reverse replication is enabled on a SwitchWrites error out if there are no tablets available on the target to source data for the reverse flow Signed-off-by: Rohit Nayak Signed-off-by: Rohit Nayak --- go/vt/discovery/tablet_picker.go | 6 +- go/vt/vtctl/vtctl.go | 9 +- .../tabletmanager/vreplication/engine.go | 7 +- go/vt/wrangler/fake_dbclient_test.go | 6 + go/vt/wrangler/resharder.go | 1 + go/vt/wrangler/traffic_switcher.go | 55 +++- go/vt/wrangler/traffic_switcher_env_test.go | 9 +- go/vt/wrangler/traffic_switcher_test.go | 297 ++++++++++++++++++ 8 files changed, 380 insertions(+), 10 deletions(-) diff --git a/go/vt/discovery/tablet_picker.go b/go/vt/discovery/tablet_picker.go index 0c72716223..af813ae13e 100644 --- a/go/vt/discovery/tablet_picker.go +++ b/go/vt/discovery/tablet_picker.go @@ -107,7 +107,7 @@ func (tp *TabletPicker) PickForStreaming(ctx context.Context) (*topodatapb.Table return nil, vterrors.Errorf(vtrpcpb.Code_CANCELED, "context has expired") default: } - candidates := tp.getMatchingTablets(ctx) + candidates := tp.GetMatchingTablets(ctx) if len(candidates) == 0 { // if no candidates were found, sleep and try again @@ -145,9 +145,9 @@ func (tp *TabletPicker) PickForStreaming(ctx context.Context) (*topodatapb.Table } } -// getMatchingTablets returns a list of TabletInfo for tablets +// GetMatchingTablets returns a list of TabletInfo for tablets // that match the cells, keyspace, shard and tabletTypes for this TabletPicker -func (tp *TabletPicker) getMatchingTablets(ctx context.Context) []*topo.TabletInfo { +func (tp *TabletPicker) GetMatchingTablets(ctx context.Context) []*topo.TabletInfo { // Special handling for MASTER tablet type // Since there is only one master, we ignore cell and find the master aliases := make([]*topodatapb.TabletAlias, 0) diff --git a/go/vt/vtctl/vtctl.go b/go/vt/vtctl/vtctl.go index 1168c87d94..4fa1c96389 100644 --- a/go/vt/vtctl/vtctl.go +++ b/go/vt/vtctl/vtctl.go @@ -2132,9 +2132,14 @@ func commandVRWorkflow(ctx context.Context, wr *wrangler.Wrangler, subFlags *fla s := "" var progress wrangler.TableCopyProgress for table := range *copyProgress { + var rowCountPct, tableSizePct int64 progress = *(*copyProgress)[table] - rowCountPct := 100.0 * progress.TargetRowCount / progress.SourceRowCount - tableSizePct := 100.0 * progress.TargetTableSize / progress.SourceTableSize + if progress.SourceRowCount > 0 { + rowCountPct = 100.0 * progress.TargetRowCount / progress.SourceRowCount + } + if progress.SourceTableSize > 0 { + tableSizePct = 100.0 * progress.TargetTableSize / progress.SourceTableSize + } s += fmt.Sprintf("%s: rows copied %d/%d (%d%%), size copied %d/%d (%d%%)\n", table, progress.TargetRowCount, progress.SourceRowCount, rowCountPct, progress.TargetTableSize, progress.SourceTableSize, tableSizePct) diff --git a/go/vt/vttablet/tabletmanager/vreplication/engine.go b/go/vt/vttablet/tabletmanager/vreplication/engine.go index 910e94d9b4..9ddf5c7547 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/engine.go +++ b/go/vt/vttablet/tabletmanager/vreplication/engine.go @@ -736,8 +736,11 @@ func (vre *Engine) WaitForPos(ctx context.Context, id int, pos string) error { select { case <-ctx.Done(): - log.Errorf("Error waiting for pos: %s, last pos: %s: %v, wait time: %v", pos, qr.Rows[0][0].ToString(), ctx.Err(), time.Since(start)) - return fmt.Errorf("error waiting for pos: %s, last pos: %s: %v, wait time: %v", pos, qr.Rows[0][0].ToString(), ctx.Err(), time.Since(start)) + err = fmt.Errorf("error waiting for pos: %s, last pos: %s: %v, wait time: %v: %s", + pos, qr.Rows[0][0].ToString(), ctx.Err(), time.Since(start), + "possibly no tablets are available to stream in the source keyspace for your cell and tablet_types setting") + log.Error(err.Error()) + return err case <-vre.ctx.Done(): return fmt.Errorf("vreplication is closing: %v", vre.ctx.Err()) case <-tkr.C: diff --git a/go/vt/wrangler/fake_dbclient_test.go b/go/vt/wrangler/fake_dbclient_test.go index 9aeef6541c..8c24f39a36 100644 --- a/go/vt/wrangler/fake_dbclient_test.go +++ b/go/vt/wrangler/fake_dbclient_test.go @@ -100,6 +100,11 @@ func (dc *fakeDBClient) addQueryRE(query string, result *sqltypes.Result, err er dc.queriesRE[query] = &dbResults{results: []*dbResult{dbr}, err: err} } +func (dc *fakeDBClient) getInvariant(query string) *sqltypes.Result { + return dc.invariants[query] +} + +// note: addInvariant will replace a previous result for a query with the provided one: this is used in the tests func (dc *fakeDBClient) addInvariant(query string, result *sqltypes.Result) { dc.invariants[query] = result } @@ -138,6 +143,7 @@ func (dc *fakeDBClient) ExecuteFetch(query string, maxrows int) (qr *sqltypes.Re if testMode == "debug" { fmt.Printf("ExecuteFetch: %s\n", query) } + if dbrs := dc.queries[query]; dbrs != nil { return dbrs.next(query) } diff --git a/go/vt/wrangler/resharder.go b/go/vt/wrangler/resharder.go index 5d856e579a..a758d84982 100644 --- a/go/vt/wrangler/resharder.go +++ b/go/vt/wrangler/resharder.go @@ -80,6 +80,7 @@ func (wr *Wrangler) Reshard(ctx context.Context, keyspace, workflow string, sour if err != nil { return vterrors.Wrap(err, "buildResharder") } + rs.stopAfterCopy = stopAfterCopy if !skipSchemaCopy { if err := rs.copySchema(ctx); err != nil { diff --git a/go/vt/wrangler/traffic_switcher.go b/go/vt/wrangler/traffic_switcher.go index 381e4bf198..e07a86b175 100644 --- a/go/vt/wrangler/traffic_switcher.go +++ b/go/vt/wrangler/traffic_switcher.go @@ -26,6 +26,8 @@ import ( "sync" "time" + "vitess.io/vitess/go/vt/discovery" + "vitess.io/vitess/go/json2" "vitess.io/vitess/go/vt/binlog/binlogplayer" "vitess.io/vitess/go/vt/concurrency" @@ -367,8 +369,52 @@ func (wr *Wrangler) SwitchReads(ctx context.Context, targetKeyspace, workflowNam return sw.logs(), nil } +func (wr *Wrangler) areTabletsAvailableToStreamFrom(ctx context.Context, ts *trafficSwitcher, keyspace string, shards []*topo.ShardInfo) error { + var cells []string + tabletTypes := ts.optTabletTypes + if ts.optCells != "" { + cells = strings.Split(ts.optCells, ",") + } + // FIXME: currently there is a default setting in the tablet that is used if user does not specify a tablet type, + // we use the value specified in the tablet flag `-vreplication_tablet_type` + // but ideally we should populate the vreplication table with a default value when we setup the workflow + if tabletTypes == "" { + tabletTypes = "MASTER,REPLICA" + } + + var wg sync.WaitGroup + allErrors := &concurrency.AllErrorRecorder{} + for _, shard := range shards { + wg.Add(1) + go func(cells []string, keyspace string, shard *topo.ShardInfo) { + defer wg.Done() + if cells == nil { + cells = append(cells, shard.MasterAlias.Cell) + } + tp, err := discovery.NewTabletPicker(wr.ts, cells, keyspace, shard.ShardName(), tabletTypes) + if err != nil { + allErrors.RecordError(err) + return + } + tablets := tp.GetMatchingTablets(ctx) + if len(tablets) == 0 { + allErrors.RecordError(fmt.Errorf("no tablet found to source data in keyspace %s, shard %s", keyspace, shard.ShardName())) + return + } + }(cells, keyspace, shard) + } + + wg.Wait() + if allErrors.HasErrors() { + log.Errorf("%s", allErrors.Error()) + return allErrors.Error() + } + return nil +} + // SwitchWrites is a generic way of migrating write traffic for a resharding workflow. -func (wr *Wrangler) SwitchWrites(ctx context.Context, targetKeyspace, workflowName string, timeout time.Duration, cancel, reverse, reverseReplication bool, dryRun bool) (journalID int64, dryRunResults *[]string, err error) { +func (wr *Wrangler) SwitchWrites(ctx context.Context, targetKeyspace, workflowName string, timeout time.Duration, + cancel, reverse, reverseReplication bool, dryRun bool) (journalID int64, dryRunResults *[]string, err error) { ts, ws, err := wr.getWorkflowState(ctx, targetKeyspace, workflowName) _ = ws if err != nil { @@ -399,6 +445,13 @@ func (wr *Wrangler) SwitchWrites(ctx context.Context, targetKeyspace, workflowNa return 0, nil, err } + if reverseReplication { + err := wr.areTabletsAvailableToStreamFrom(ctx, ts, ts.targetKeyspace, ts.targetShards()) + if err != nil { + return 0, nil, err + } + } + // Need to lock both source and target keyspaces. tctx, sourceUnlock, lockErr := sw.lockKeyspace(ctx, ts.sourceKeyspace, "SwitchWrites") if lockErr != nil { diff --git a/go/vt/wrangler/traffic_switcher_env_test.go b/go/vt/wrangler/traffic_switcher_env_test.go index d9e49802f8..e9c1a8f9f2 100644 --- a/go/vt/wrangler/traffic_switcher_env_test.go +++ b/go/vt/wrangler/traffic_switcher_env_test.go @@ -314,9 +314,8 @@ func newTestShardMigrater(ctx context.Context, t *testing.T, sourceShards, targe tme.startTablets(t) tme.createDBClients(ctx, t) tme.setMasterPositions() - for i, targetShard := range targetShards { - var rows []string + var rows, rowsRdOnly []string for j, sourceShard := range sourceShards { if !key.KeyRangesIntersect(tme.targetKeyRanges[i], tme.sourceKeyRanges[j]) { continue @@ -332,12 +331,18 @@ func newTestShardMigrater(ctx context.Context, t *testing.T, sourceShards, targe }, } rows = append(rows, fmt.Sprintf("%d|%v|||", j+1, bls)) + rowsRdOnly = append(rows, fmt.Sprintf("%d|%v|||RDONLY", j+1, bls)) } tme.dbTargetClients[i].addInvariant(vreplQueryks, sqltypes.MakeTestResult(sqltypes.MakeTestFields( "id|source|message|cell|tablet_types", "int64|varchar|varchar|varchar|varchar"), rows...), ) + tme.dbTargetClients[i].addInvariant(vreplQueryks+"-rdonly", sqltypes.MakeTestResult(sqltypes.MakeTestFields( + "id|source|message|cell|tablet_types", + "int64|varchar|varchar|varchar|varchar"), + rowsRdOnly...), + ) } tme.targetKeyspace = "ks" diff --git a/go/vt/wrangler/traffic_switcher_test.go b/go/vt/wrangler/traffic_switcher_test.go index 4241c2fbd3..22cfcaa0c0 100644 --- a/go/vt/wrangler/traffic_switcher_test.go +++ b/go/vt/wrangler/traffic_switcher_test.go @@ -1735,6 +1735,303 @@ func TestReverseVReplicationUpdateQuery(t *testing.T) { } } +func TestShardMigrateNoAvailableTabletsForReverseReplication(t *testing.T) { + ctx := context.Background() + tme := newTestShardMigrater(ctx, t, []string{"-40", "40-"}, []string{"-80", "80-"}) + defer tme.stopTablets(t) + + // Initial check + checkServedTypes(t, tme.ts, "ks:-40", 3) + checkServedTypes(t, tme.ts, "ks:40-", 3) + checkServedTypes(t, tme.ts, "ks:-80", 0) + checkServedTypes(t, tme.ts, "ks:80-", 0) + + tme.expectNoPreviousJournals() + //------------------------------------------------------------------------------------------------------------------- + // Single cell RDONLY migration. + _, err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_RDONLY}, []string{"cell1"}, DirectionForward, false) + if err != nil { + t.Fatal(err) + } + checkCellServedTypes(t, tme.ts, "ks:-40", "cell1", 2) + checkCellServedTypes(t, tme.ts, "ks:40-", "cell1", 2) + checkCellServedTypes(t, tme.ts, "ks:-80", "cell1", 1) + checkCellServedTypes(t, tme.ts, "ks:80-", "cell1", 1) + checkCellServedTypes(t, tme.ts, "ks:-40", "cell2", 3) + checkCellServedTypes(t, tme.ts, "ks:40-", "cell2", 3) + checkCellServedTypes(t, tme.ts, "ks:-80", "cell2", 0) + checkCellServedTypes(t, tme.ts, "ks:80-", "cell2", 0) + verifyQueries(t, tme.allDBClients) + + tme.expectNoPreviousJournals() + //------------------------------------------------------------------------------------------------------------------- + // Other cell REPLICA migration. + _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_REPLICA}, []string{"cell2"}, DirectionForward, false) + if err != nil { + t.Fatal(err) + } + checkCellServedTypes(t, tme.ts, "ks:-40", "cell1", 2) + checkCellServedTypes(t, tme.ts, "ks:40-", "cell1", 2) + checkCellServedTypes(t, tme.ts, "ks:-80", "cell1", 1) + checkCellServedTypes(t, tme.ts, "ks:80-", "cell1", 1) + checkCellServedTypes(t, tme.ts, "ks:-40", "cell2", 1) + checkCellServedTypes(t, tme.ts, "ks:40-", "cell2", 1) + checkCellServedTypes(t, tme.ts, "ks:-80", "cell2", 2) + checkCellServedTypes(t, tme.ts, "ks:80-", "cell2", 2) + verifyQueries(t, tme.allDBClients) + + tme.expectNoPreviousJournals() + //------------------------------------------------------------------------------------------------------------------- + // Single cell backward REPLICA migration. + _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_REPLICA}, []string{"cell2"}, DirectionBackward, false) + if err != nil { + t.Fatal(err) + } + checkCellServedTypes(t, tme.ts, "ks:-40", "cell1", 2) + checkCellServedTypes(t, tme.ts, "ks:40-", "cell1", 2) + checkCellServedTypes(t, tme.ts, "ks:-80", "cell1", 1) + checkCellServedTypes(t, tme.ts, "ks:80-", "cell1", 1) + checkCellServedTypes(t, tme.ts, "ks:-40", "cell2", 3) + checkCellServedTypes(t, tme.ts, "ks:40-", "cell2", 3) + checkCellServedTypes(t, tme.ts, "ks:-80", "cell2", 0) + checkCellServedTypes(t, tme.ts, "ks:80-", "cell2", 0) + verifyQueries(t, tme.allDBClients) + + tme.expectNoPreviousJournals() + //------------------------------------------------------------------------------------------------------------------- + // Switch all RDONLY. + // This is an extra step that does not exist in the tables test. + // The per-cell migration mechanism is different for tables. So, this + // extra step is needed to bring things in sync. + _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_RDONLY}, nil, DirectionForward, false) + if err != nil { + t.Fatal(err) + } + checkServedTypes(t, tme.ts, "ks:-40", 2) + checkServedTypes(t, tme.ts, "ks:40-", 2) + checkServedTypes(t, tme.ts, "ks:-80", 1) + checkServedTypes(t, tme.ts, "ks:80-", 1) + verifyQueries(t, tme.allDBClients) + + tme.expectNoPreviousJournals() + //------------------------------------------------------------------------------------------------------------------- + // Switch all REPLICA. + _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_REPLICA}, nil, DirectionForward, false) + if err != nil { + t.Fatal(err) + } + checkServedTypes(t, tme.ts, "ks:-40", 1) + checkServedTypes(t, tme.ts, "ks:40-", 1) + checkServedTypes(t, tme.ts, "ks:-80", 2) + checkServedTypes(t, tme.ts, "ks:80-", 2) + verifyQueries(t, tme.allDBClients) + + tme.expectNoPreviousJournals() + //------------------------------------------------------------------------------------------------------------------- + // All cells RDONLY backward migration. + _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_RDONLY}, nil, DirectionBackward, false) + if err != nil { + t.Fatal(err) + } + checkServedTypes(t, tme.ts, "ks:-40", 2) + checkServedTypes(t, tme.ts, "ks:40-", 2) + checkServedTypes(t, tme.ts, "ks:-80", 1) + checkServedTypes(t, tme.ts, "ks:80-", 1) + verifyQueries(t, tme.allDBClients) + + //------------------------------------------------------------------------------------------------------------------- + // Can't switch master with SwitchReads. + _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_MASTER}, nil, DirectionForward, false) + want := "tablet type must be REPLICA or RDONLY: MASTER" + if err == nil || err.Error() != want { + t.Errorf("SwitchReads(master) err: %v, want %v", err, want) + } + verifyQueries(t, tme.allDBClients) + + //------------------------------------------------------------------------------------------------------------------- + // Test SwitchWrites cancelation on failure. + + tme.expectNoPreviousJournals() + // Switch all the reads first. + _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_RDONLY}, nil, DirectionForward, false) + if err != nil { + t.Fatal(err) + } + checkServedTypes(t, tme.ts, "ks:-40", 1) + checkServedTypes(t, tme.ts, "ks:40-", 1) + checkServedTypes(t, tme.ts, "ks:-80", 2) + checkServedTypes(t, tme.ts, "ks:80-", 2) + checkIsMasterServing(t, tme.ts, "ks:-40", true) + checkIsMasterServing(t, tme.ts, "ks:40-", true) + checkIsMasterServing(t, tme.ts, "ks:-80", false) + checkIsMasterServing(t, tme.ts, "ks:80-", false) + + checkJournals := func() { + tme.dbSourceClients[0].addQuery("select val from _vt.resharding_journal where id=6432976123657117097", &sqltypes.Result{}, nil) + tme.dbSourceClients[1].addQuery("select val from _vt.resharding_journal where id=6432976123657117097", &sqltypes.Result{}, nil) + } + checkJournals() + + stopStreams := func() { + tme.dbSourceClients[0].addQuery("select id, workflow, source, pos from _vt.vreplication where db_name='vt_ks' and workflow != 'test_reverse' and state = 'Stopped' and message != 'FROZEN'", &sqltypes.Result{}, nil) + tme.dbSourceClients[1].addQuery("select id, workflow, source, pos from _vt.vreplication where db_name='vt_ks' and workflow != 'test_reverse' and state = 'Stopped' and message != 'FROZEN'", &sqltypes.Result{}, nil) + tme.dbSourceClients[0].addQuery("select id, workflow, source, pos from _vt.vreplication where db_name='vt_ks' and workflow != 'test_reverse'", &sqltypes.Result{}, nil) + tme.dbSourceClients[1].addQuery("select id, workflow, source, pos from _vt.vreplication where db_name='vt_ks' and workflow != 'test_reverse'", &sqltypes.Result{}, nil) + } + stopStreams() + + deleteReverseReplicaion := func() { + tme.dbSourceClients[0].addQuery("select id from _vt.vreplication where db_name = 'vt_ks' and workflow = 'test_reverse'", resultid3, nil) + tme.dbSourceClients[1].addQuery("select id from _vt.vreplication where db_name = 'vt_ks' and workflow = 'test_reverse'", resultid34, nil) + tme.dbSourceClients[0].addQuery("delete from _vt.vreplication where id in (3)", &sqltypes.Result{}, nil) + tme.dbSourceClients[1].addQuery("delete from _vt.vreplication where id in (3, 4)", &sqltypes.Result{}, nil) + tme.dbSourceClients[0].addQuery("delete from _vt.copy_state where vrepl_id in (3)", &sqltypes.Result{}, nil) + tme.dbSourceClients[1].addQuery("delete from _vt.copy_state where vrepl_id in (3, 4)", &sqltypes.Result{}, nil) + } + cancelMigration := func() { + tme.dbSourceClients[0].addQuery("select id from _vt.vreplication where db_name = 'vt_ks' and workflow != 'test_reverse'", &sqltypes.Result{}, nil) + tme.dbSourceClients[1].addQuery("select id from _vt.vreplication where db_name = 'vt_ks' and workflow != 'test_reverse'", &sqltypes.Result{}, nil) + + tme.dbTargetClients[0].addQuery("select id from _vt.vreplication where db_name = 'vt_ks' and workflow = 'test'", resultid12, nil) + tme.dbTargetClients[1].addQuery("select id from _vt.vreplication where db_name = 'vt_ks' and workflow = 'test'", resultid2, nil) + tme.dbTargetClients[0].addQuery("update _vt.vreplication set state = 'Running', message = '' where id in (1, 2)", &sqltypes.Result{}, nil) + tme.dbTargetClients[1].addQuery("update _vt.vreplication set state = 'Running', message = '' where id in (2)", &sqltypes.Result{}, nil) + tme.dbTargetClients[0].addQuery("select * from _vt.vreplication where id = 1", runningResult(1), nil) + tme.dbTargetClients[0].addQuery("select * from _vt.vreplication where id = 2", runningResult(2), nil) + tme.dbTargetClients[1].addQuery("select * from _vt.vreplication where id = 2", runningResult(2), nil) + + deleteReverseReplicaion() + } + cancelMigration() + + _, _, err = tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 0*time.Second, false, false, true, false) + want = "DeadlineExceeded" + if err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("SwitchWrites(0 timeout) err: %v, must contain %v", err, want) + } + + verifyQueries(t, tme.allDBClients) + checkServedTypes(t, tme.ts, "ks:-40", 1) + checkServedTypes(t, tme.ts, "ks:40-", 1) + checkServedTypes(t, tme.ts, "ks:-80", 2) + checkServedTypes(t, tme.ts, "ks:80-", 2) + checkIsMasterServing(t, tme.ts, "ks:-40", true) + checkIsMasterServing(t, tme.ts, "ks:40-", true) + checkIsMasterServing(t, tme.ts, "ks:-80", false) + checkIsMasterServing(t, tme.ts, "ks:80-", false) + + //------------------------------------------------------------------------------------------------------------------- + // Test successful SwitchWrites. + + checkJournals() + stopStreams() + + waitForCatchup := func() { + // mi.waitForCatchup-> mi.wr.tmc.VReplicationWaitForPos + state := sqltypes.MakeTestResult(sqltypes.MakeTestFields( + "pos|state|message", + "varchar|varchar|varchar"), + "MariaDB/5-456-892|Running", + ) + tme.dbTargetClients[0].addQuery("select pos, state, message from _vt.vreplication where id=1", state, nil) + tme.dbTargetClients[1].addQuery("select pos, state, message from _vt.vreplication where id=2", state, nil) + tme.dbTargetClients[0].addQuery("select pos, state, message from _vt.vreplication where id=2", state, nil) + + // mi.waitForCatchup-> mi.wr.tmc.VReplicationExec('stopped for cutover') + tme.dbTargetClients[0].addQuery("select id from _vt.vreplication where id = 1", resultid1, nil) + tme.dbTargetClients[0].addQuery("update _vt.vreplication set state = 'Stopped', message = 'stopped for cutover' where id in (1)", &sqltypes.Result{}, nil) + tme.dbTargetClients[0].addQuery("select id from _vt.vreplication where id = 2", resultid2, nil) + tme.dbTargetClients[0].addQuery("update _vt.vreplication set state = 'Stopped', message = 'stopped for cutover' where id in (2)", &sqltypes.Result{}, nil) + tme.dbTargetClients[1].addQuery("select id from _vt.vreplication where id = 2", resultid2, nil) + tme.dbTargetClients[1].addQuery("update _vt.vreplication set state = 'Stopped', message = 'stopped for cutover' where id in (2)", &sqltypes.Result{}, nil) + tme.dbTargetClients[0].addQuery("select * from _vt.vreplication where id = 1", stoppedResult(1), nil) + tme.dbTargetClients[1].addQuery("select * from _vt.vreplication where id = 2", stoppedResult(2), nil) + tme.dbTargetClients[0].addQuery("select * from _vt.vreplication where id = 2", stoppedResult(2), nil) + } + waitForCatchup() + + createReverseVReplication := func() { + deleteReverseReplicaion() + + tme.dbSourceClients[0].addQueryRE("insert into _vt.vreplication.*-80.*-40.*MariaDB/5-456-893.*Stopped", &sqltypes.Result{InsertID: 1}, nil) + tme.dbSourceClients[1].addQueryRE("insert into _vt.vreplication.*-80.*40-.*MariaDB/5-456-893.*Stopped", &sqltypes.Result{InsertID: 1}, nil) + tme.dbSourceClients[1].addQueryRE("insert into _vt.vreplication.*80-.*40-.*MariaDB/5-456-893.*Stopped", &sqltypes.Result{InsertID: 2}, nil) + tme.dbSourceClients[0].addQuery("select * from _vt.vreplication where id = 1", stoppedResult(1), nil) + tme.dbSourceClients[1].addQuery("select * from _vt.vreplication where id = 1", stoppedResult(1), nil) + tme.dbSourceClients[1].addQuery("select * from _vt.vreplication where id = 2", stoppedResult(2), nil) + } + createReverseVReplication() + + createJournals := func() { + journal1 := "insert into _vt.resharding_journal.*6432976123657117097.*migration_type:SHARDS.*local_position.*MariaDB/5-456-892.*shard_gtids.*-80.*MariaDB/5-456-893.*participants.*40.*40" + tme.dbSourceClients[0].addQueryRE(journal1, &sqltypes.Result{}, nil) + journal2 := "insert into _vt.resharding_journal.*6432976123657117097.*migration_type:SHARDS.*local_position.*MariaDB/5-456-892.*shard_gtids.*80.*MariaDB/5-456-893.*shard_gtids.*80.*MariaDB/5-456-893.*participants.*40.*40" + tme.dbSourceClients[1].addQueryRE(journal2, &sqltypes.Result{}, nil) + } + createJournals() + + startReverseVReplication := func() { + tme.dbSourceClients[0].addQuery("select id from _vt.vreplication where db_name = 'vt_ks'", resultid34, nil) + tme.dbSourceClients[0].addQuery("update _vt.vreplication set state = 'Running', message = '' where id in (3, 4)", &sqltypes.Result{}, nil) + tme.dbSourceClients[0].addQuery("select * from _vt.vreplication where id = 3", runningResult(3), nil) + tme.dbSourceClients[0].addQuery("select * from _vt.vreplication where id = 4", runningResult(4), nil) + tme.dbSourceClients[1].addQuery("select id from _vt.vreplication where db_name = 'vt_ks'", resultid34, nil) + tme.dbSourceClients[1].addQuery("update _vt.vreplication set state = 'Running', message = '' where id in (3, 4)", &sqltypes.Result{}, nil) + tme.dbSourceClients[1].addQuery("select * from _vt.vreplication where id = 3", runningResult(3), nil) + tme.dbSourceClients[1].addQuery("select * from _vt.vreplication where id = 4", runningResult(4), nil) + } + startReverseVReplication() + + freezeTargetVReplication := func() { + tme.dbTargetClients[0].addQuery("select id from _vt.vreplication where db_name = 'vt_ks' and workflow = 'test'", resultid12, nil) + tme.dbTargetClients[0].addQuery("update _vt.vreplication set message = 'FROZEN' where id in (1, 2)", &sqltypes.Result{}, nil) + tme.dbTargetClients[0].addQuery("select * from _vt.vreplication where id = 1", stoppedResult(1), nil) + tme.dbTargetClients[0].addQuery("select * from _vt.vreplication where id = 2", stoppedResult(2), nil) + tme.dbTargetClients[1].addQuery("select id from _vt.vreplication where db_name = 'vt_ks' and workflow = 'test'", resultid2, nil) + tme.dbTargetClients[1].addQuery("update _vt.vreplication set message = 'FROZEN' where id in (2)", &sqltypes.Result{}, nil) + tme.dbTargetClients[1].addQuery("select * from _vt.vreplication where id = 2", stoppedResult(2), nil) + } + freezeTargetVReplication() + + // Temporarily set tablet types to RDONLY to test that SwitchWrites fails if no tablets of rdonly are available + invariants := make(map[string]*sqltypes.Result) + for i := range tme.targetShards { + invariants[fmt.Sprintf("%s-%d", vreplQueryks, i)] = tme.dbTargetClients[i].getInvariant(vreplQueryks) + tme.dbTargetClients[i].addInvariant(vreplQueryks, tme.dbTargetClients[i].getInvariant(vreplQueryks+"-rdonly")) + } + _, _, err = tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, false, true, false) + require.Error(t, err) + require.True(t, strings.Contains(err.Error(), "no tablet found")) + require.True(t, strings.Contains(err.Error(), "-80")) + require.True(t, strings.Contains(err.Error(), "80-")) + require.False(t, strings.Contains(err.Error(), "40")) + for i := range tme.targetShards { + tme.dbTargetClients[i].addInvariant(vreplQueryks, invariants[fmt.Sprintf("%s-%d", vreplQueryks, i)]) + } + + journalID, _, err := tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, false, true, false) + if err != nil { + t.Fatal(err) + } + if journalID != 6432976123657117097 { + t.Errorf("journal id: %d, want 6432976123657117097", journalID) + } + + verifyQueries(t, tme.allDBClients) + + checkServedTypes(t, tme.ts, "ks:-40", 0) + checkServedTypes(t, tme.ts, "ks:40-", 0) + checkServedTypes(t, tme.ts, "ks:-80", 3) + checkServedTypes(t, tme.ts, "ks:80-", 3) + + checkIsMasterServing(t, tme.ts, "ks:-40", false) + checkIsMasterServing(t, tme.ts, "ks:40-", false) + checkIsMasterServing(t, tme.ts, "ks:-80", true) + checkIsMasterServing(t, tme.ts, "ks:80-", true) + + verifyQueries(t, tme.allDBClients) +} + func checkRouting(t *testing.T, wr *Wrangler, want map[string][]string) { t.Helper() ctx := context.Background() From 94ce516a9f000c4770c4dde68278e678fd973a0a Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Mon, 7 Jun 2021 22:31:15 +0200 Subject: [PATCH 34/64] Fix merge issue Signed-off-by: Rohit Nayak --- go/vt/wrangler/traffic_switcher_test.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/go/vt/wrangler/traffic_switcher_test.go b/go/vt/wrangler/traffic_switcher_test.go index 22cfcaa0c0..21c21e57db 100644 --- a/go/vt/wrangler/traffic_switcher_test.go +++ b/go/vt/wrangler/traffic_switcher_test.go @@ -1749,7 +1749,7 @@ func TestShardMigrateNoAvailableTabletsForReverseReplication(t *testing.T) { tme.expectNoPreviousJournals() //------------------------------------------------------------------------------------------------------------------- // Single cell RDONLY migration. - _, err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_RDONLY}, []string{"cell1"}, DirectionForward, false) + _, err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_RDONLY}, []string{"cell1"}, workflow.DirectionForward, false) if err != nil { t.Fatal(err) } @@ -1766,7 +1766,7 @@ func TestShardMigrateNoAvailableTabletsForReverseReplication(t *testing.T) { tme.expectNoPreviousJournals() //------------------------------------------------------------------------------------------------------------------- // Other cell REPLICA migration. - _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_REPLICA}, []string{"cell2"}, DirectionForward, false) + _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_REPLICA}, []string{"cell2"}, workflow.DirectionForward, false) if err != nil { t.Fatal(err) } @@ -1783,7 +1783,7 @@ func TestShardMigrateNoAvailableTabletsForReverseReplication(t *testing.T) { tme.expectNoPreviousJournals() //------------------------------------------------------------------------------------------------------------------- // Single cell backward REPLICA migration. - _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_REPLICA}, []string{"cell2"}, DirectionBackward, false) + _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_REPLICA}, []string{"cell2"}, workflow.DirectionBackward, false) if err != nil { t.Fatal(err) } @@ -1803,7 +1803,7 @@ func TestShardMigrateNoAvailableTabletsForReverseReplication(t *testing.T) { // This is an extra step that does not exist in the tables test. // The per-cell migration mechanism is different for tables. So, this // extra step is needed to bring things in sync. - _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_RDONLY}, nil, DirectionForward, false) + _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_RDONLY}, nil, workflow.DirectionForward, false) if err != nil { t.Fatal(err) } @@ -1816,7 +1816,7 @@ func TestShardMigrateNoAvailableTabletsForReverseReplication(t *testing.T) { tme.expectNoPreviousJournals() //------------------------------------------------------------------------------------------------------------------- // Switch all REPLICA. - _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_REPLICA}, nil, DirectionForward, false) + _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_REPLICA}, nil, workflow.DirectionForward, false) if err != nil { t.Fatal(err) } @@ -1829,7 +1829,7 @@ func TestShardMigrateNoAvailableTabletsForReverseReplication(t *testing.T) { tme.expectNoPreviousJournals() //------------------------------------------------------------------------------------------------------------------- // All cells RDONLY backward migration. - _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_RDONLY}, nil, DirectionBackward, false) + _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_RDONLY}, nil, workflow.DirectionBackward, false) if err != nil { t.Fatal(err) } @@ -1841,7 +1841,7 @@ func TestShardMigrateNoAvailableTabletsForReverseReplication(t *testing.T) { //------------------------------------------------------------------------------------------------------------------- // Can't switch master with SwitchReads. - _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_MASTER}, nil, DirectionForward, false) + _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_MASTER}, nil, workflow.DirectionForward, false) want := "tablet type must be REPLICA or RDONLY: MASTER" if err == nil || err.Error() != want { t.Errorf("SwitchReads(master) err: %v, want %v", err, want) @@ -1853,7 +1853,7 @@ func TestShardMigrateNoAvailableTabletsForReverseReplication(t *testing.T) { tme.expectNoPreviousJournals() // Switch all the reads first. - _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_RDONLY}, nil, DirectionForward, false) + _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_RDONLY}, nil, workflow.DirectionForward, false) if err != nil { t.Fatal(err) } From 190daf1499465c4aa2352ee5af0c03bc8f7f558f Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Tue, 8 Jun 2021 09:11:30 +0530 Subject: [PATCH 35/64] checked loaded in updateController inside mutex, skip processing for non-primary upfront Signed-off-by: Harshit Gangal --- go/vt/vtgate/schema/tracker.go | 4 ++-- go/vt/vtgate/schema/uptate_controller.go | 21 +++++++++++++++---- go/vt/vtgate/schema/uptate_controller_test.go | 5 +++-- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/go/vt/vtgate/schema/tracker.go b/go/vt/vtgate/schema/tracker.go index 081907f691..f819bbcd36 100644 --- a/go/vt/vtgate/schema/tracker.go +++ b/go/vt/vtgate/schema/tracker.go @@ -76,7 +76,7 @@ func (t *Tracker) LoadKeyspace(conn queryservice.QueryService, target *querypb.T t.mu.Lock() defer t.mu.Unlock() t.updateTables(target.Keyspace, res) - t.tracked[target.Keyspace].loaded = true + t.tracked[target.Keyspace].setLoaded(true) log.Infof("finished loading schema for keyspace %s. Found %d tables", target.Keyspace, len(res.Rows)) return nil } @@ -164,7 +164,7 @@ func (t *Tracker) updateSchema(th *discovery.TabletHealth) bool { bv := map[string]*querypb.BindVariable{"tableNames": tables} res, err := th.Conn.Execute(t.ctx, th.Target, mysql.FetchUpdatedTables, bv, 0, 0, nil) if err != nil { - t.tracked[th.Target.Keyspace].loaded = false + t.tracked[th.Target.Keyspace].setLoaded(false) // TODO: optimize for the tables that got errored out. log.Warningf("error fetching new schema for %v, making them non-authoritative: %v", tablesUpdated, err) return false diff --git a/go/vt/vtgate/schema/uptate_controller.go b/go/vt/vtgate/schema/uptate_controller.go index 7d6b3a7d7a..27f02edf07 100644 --- a/go/vt/vtgate/schema/uptate_controller.go +++ b/go/vt/vtgate/schema/uptate_controller.go @@ -94,22 +94,35 @@ func (u *updateController) getItemFromQueueLocked() *discovery.TabletHealth { } func (u *updateController) add(th *discovery.TabletHealth) { - u.mu.Lock() - defer u.mu.Unlock() + // For non-primary tablet health, there is no schema tracking. + if th.Tablet.Type != topodatapb.TabletType_MASTER { + return + } - // Received a health check from primary tablet that is it not reachable from VTGate. + // Received a health check from primary tablet that is not reachable from VTGate. // The connection will get reset and the tracker needs to reload the schema for the keyspace. - if th.Tablet.Type == topodatapb.TabletType_MASTER && !th.Serving { + if !th.Serving { u.loaded = false return } + u.mu.Lock() + defer u.mu.Unlock() + + // If the keyspace schema is loaded and there is no schema change detected. Then there is nothing to process. if len(th.Stats.TableSchemaChanged) == 0 && u.loaded { return } + if u.queue == nil { u.queue = &queue{} go u.consume() } u.queue.items = append(u.queue.items, th) } + +func (u *updateController) setLoaded(loaded bool) { + u.mu.Lock() + defer u.mu.Unlock() + u.loaded = loaded +} diff --git a/go/vt/vtgate/schema/uptate_controller_test.go b/go/vt/vtgate/schema/uptate_controller_test.go index 04895fccdf..8521cbe163 100644 --- a/go/vt/vtgate/schema/uptate_controller_test.go +++ b/go/vt/vtgate/schema/uptate_controller_test.go @@ -137,8 +137,9 @@ func TestMultipleUpdatesFromDifferentShards(t *testing.T) { for _, in := range test.inputs { target := &querypb.Target{ - Keyspace: "ks", - Shard: in.shard, + Keyspace: "ks", + Shard: in.shard, + TabletType: topodatapb.TabletType_MASTER, } tablet := &topodatapb.Tablet{ Keyspace: target.Keyspace, From e6bc4bfbcf510df1b97aea8e411ba38636c5bf79 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Tue, 8 Jun 2021 13:31:21 +0200 Subject: [PATCH 36/64] Ignore unrelated gtid progress events that can race with the events that the schema versioning test expects Signed-off-by: Rohit Nayak --- go/vt/vttablet/endtoend/vstreamer_test.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/go/vt/vttablet/endtoend/vstreamer_test.go b/go/vt/vttablet/endtoend/vstreamer_test.go index 9dba6a4510..cacb2e102b 100644 --- a/go/vt/vttablet/endtoend/vstreamer_test.go +++ b/go/vt/vttablet/endtoend/vstreamer_test.go @@ -357,6 +357,13 @@ func expectLogs(ctx context.Context, t *testing.T, query string, eventCh chan [] if !ok { t.Fatal("expectLogs: not ok, stream ended early") } + // Ignore unrelated gtid progress events that can race with the events that the test expects + if len(allevs) == 3 && + allevs[0].Type == binlogdatapb.VEventType_BEGIN && + allevs[1].Type == binlogdatapb.VEventType_GTID && + allevs[2].Type == binlogdatapb.VEventType_COMMIT { + continue + } for _, ev := range allevs { // Ignore spurious heartbeats that can happen on slow machines. if ev.Type == binlogdatapb.VEventType_HEARTBEAT { From 732a088431f12aebe81d24154fc03b8c2dd7b85a Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 8 Jun 2021 10:26:25 -0400 Subject: [PATCH 37/64] [vtctldclient] Update help text for legacy shim Also remove the custom help func hack, because it turns out we don't actually need it (thank goodness) Signed-off-by: Andrew Mason --- .../internal/command/legacy_shim.go | 55 ++++++++++--------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/go/cmd/vtctldclient/internal/command/legacy_shim.go b/go/cmd/vtctldclient/internal/command/legacy_shim.go index 0d6bd8494b..f4036fd26a 100644 --- a/go/cmd/vtctldclient/internal/command/legacy_shim.go +++ b/go/cmd/vtctldclient/internal/command/legacy_shim.go @@ -23,7 +23,6 @@ import ( "strings" "github.com/spf13/cobra" - "k8s.io/apimachinery/pkg/util/sets" "vitess.io/vitess/go/cmd/vtctldclient/cli" "vitess.io/vitess/go/vt/log" @@ -38,13 +37,41 @@ var ( // RPCs. This allows users to use a single binary to make RPCs against both // the new and old vtctld gRPC APIs. LegacyVtctlCommand = &cobra.Command{ - Use: "LegacyVtctlCommand", + Use: "LegacyVtctlCommand -- [flags ...] [args ...]", Short: "Invoke a legacy vtctlclient command. Flag parsing is best effort.", Args: cobra.ArbitraryArgs, RunE: func(cmd *cobra.Command, args []string) error { cli.FinishedParsing(cmd) return runLegacyCommand(args) }, + Long: strings.TrimSpace(` +LegacyVtctlCommand uses the legacy vtctl grpc client to make an ExecuteVtctlCommand +rpc to a vtctld. + +This command exists to support a smooth transition of any scripts that relied on +vtctlclient during the migration to the new vtctldclient, and will be removed, +following the Vitess project's standard deprecation cycle, once all commands +have been migrated to the new VtctldServer api. + +To see the list of available legacy commands, run "LegacyVtctlCommand -- help". +Note that, as with the old client, this requires a running server, as the flag +parsing and help/usage text generation, is done server-side. + +Also note that, in order to defer that flag parsing to the server side, you must +use the double-dash ("--") after the LegacyVtctlCommand subcommand string, or +the client-side flag parsing library we are using will attempt to parse those +flags (and fail). +`), + Example: strings.TrimSpace(` +LegacyVtctlCommand help # displays this help message +LegacyVtctlCommand -- help # displays help for supported legacy vtctl commands + +# When using legacy command that take arguments, a double dash must be used +# before the first flag argument, like in the first example. The double dash may +# be used, however, at any point after the "LegacyVtctlCommand" string, as in +# the second example. +LegacyVtctlCommand AddCellInfo -- -server_address "localhost:1234" -root "/vitess/cell1" +LegacyVtctlCommand -- AddCellInfo -server_address "localhost:5678" -root "/vitess/cell1"`), } ) @@ -73,29 +100,5 @@ func runLegacyCommand(args []string) error { } func init() { - LegacyVtctlCommand.SetHelpFunc(func(cmd *cobra.Command, args []string) { - // PreRun (and PersistentPreRun) do not run when a Help command is - // being executed, so we need to duplicate the `--server` flag check - // here before we attempt to invoke the legacy help command. - if err := ensureServerArg(); err != nil { - log.Error(err) - return - } - - realArgs := cmd.Flags().Args() - - if len(realArgs) == 0 { - realArgs = append(realArgs, "help") - } - - argSet := sets.NewString(realArgs...) - if !argSet.HasAny("help", "-h", "--help") { - // Cobra tends to swallow the help flag, so we need to put it back - // into the arg slice that we pass to runLegacyCommand. - realArgs = append(realArgs, "-h") - } - - _ = runLegacyCommand(realArgs) - }) Root.AddCommand(LegacyVtctlCommand) } From 7827fff5e8f18bcce6a61a227d96ca95be09e1f7 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Sat, 5 Jun 2021 16:38:23 -0400 Subject: [PATCH 38/64] [vtctldserver] wip -- start tracing all vtctldserver endpoints Signed-off-by: Andrew Mason --- go/vt/topo/topoproto/keyspace.go | 22 ++++++++ go/vt/vtctl/grpcvtctldserver/server.go | 73 ++++++++++++++++++++++++++ go/vt/vtctl/grpcvtctldserver/topo.go | 9 ++++ 3 files changed, 104 insertions(+) diff --git a/go/vt/topo/topoproto/keyspace.go b/go/vt/topo/topoproto/keyspace.go index 625ec0b729..d40c869882 100644 --- a/go/vt/topo/topoproto/keyspace.go +++ b/go/vt/topo/topoproto/keyspace.go @@ -42,3 +42,25 @@ func KeyspaceTypeString(kt topodatapb.KeyspaceType) string { return str } + +// KeyspaceTypeLString returns the lowercased string representation of a +// KeyspaceType. +func KeyspaceTypeLString(kt topodatapb.KeyspaceType) string { + return strings.ToLower(KeyspaceTypeString(kt)) +} + +// KeyspaceIDTypeString returns the string representation of a KeyspaceIdType. +func KeyspaceIDTypeString(kidType topodatapb.KeyspaceIdType) string { + str, ok := topodatapb.KeyspaceIdType_name[int32(kidType)] + if !ok { + return "UNKNOWN" + } + + return str +} + +// KeyspaceIDTypeLString returns the lowercased string representation of a +// KeyspaceIdType. +func KeyspaceIDTypeLString(kidType topodatapb.KeyspaceIdType) string { + return strings.ToLower(KeyspaceIDTypeString(kidType)) +} diff --git a/go/vt/vtctl/grpcvtctldserver/server.go b/go/vt/vtctl/grpcvtctldserver/server.go index 55beb6a9b1..40ef1997fd 100644 --- a/go/vt/vtctl/grpcvtctldserver/server.go +++ b/go/vt/vtctl/grpcvtctldserver/server.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "path/filepath" + "strings" "sync" "time" @@ -84,10 +85,17 @@ func NewVtctldServer(ts *topo.Server) *VtctldServer { // AddCellInfo is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) AddCellInfo(ctx context.Context, req *vtctldatapb.AddCellInfoRequest) (*vtctldatapb.AddCellInfoResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.AddCellInfo") + defer span.Finish() + if req.CellInfo.Root == "" { return nil, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "CellInfo.Root must be non-empty") } + span.Annotate("cell", req.Name) + span.Annotate("cell_root", req.CellInfo.Root) + span.Annotate("cell_address", req.CellInfo.ServerAddress) + ctx, cancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout) defer cancel() @@ -100,6 +108,12 @@ func (s *VtctldServer) AddCellInfo(ctx context.Context, req *vtctldatapb.AddCell // AddCellsAlias is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) AddCellsAlias(ctx context.Context, req *vtctldatapb.AddCellsAliasRequest) (*vtctldatapb.AddCellsAliasResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.AddCellsAlias") + defer span.Finish() + + span.Annotate("cells_alias", req.Name) + span.Annotate("cells", strings.Join(req.Cells, ",")) + ctx, cancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout) defer cancel() @@ -112,6 +126,12 @@ func (s *VtctldServer) AddCellsAlias(ctx context.Context, req *vtctldatapb.AddCe // ApplyRoutingRules is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) ApplyRoutingRules(ctx context.Context, req *vtctldatapb.ApplyRoutingRulesRequest) (*vtctldatapb.ApplyRoutingRulesResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.ApplyRoutingRules") + defer span.Finish() + + span.Annotate("skip_rebuild", req.SkipRebuild) + span.Annotate("rebuild_cells", strings.Join(req.RebuildCells, ",")) + if err := s.ts.SaveRoutingRules(ctx, req.RoutingRules); err != nil { return nil, err } @@ -191,6 +211,13 @@ func (s *VtctldServer) ApplyVSchema(ctx context.Context, req *vtctldatapb.ApplyV // ChangeTabletType is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) ChangeTabletType(ctx context.Context, req *vtctldatapb.ChangeTabletTypeRequest) (*vtctldatapb.ChangeTabletTypeResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.ChangeTabletType") + defer span.Finish() + + span.Annotate("tablet_alias", topoproto.TabletAliasString(req.TabletAlias)) + span.Annotate("dry_run", req.DryRun) + span.Annotate("tablet_type", topoproto.TabletTypeLString(req.DbType)) + ctx, cancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout) defer cancel() @@ -199,6 +226,8 @@ func (s *VtctldServer) ChangeTabletType(ctx context.Context, req *vtctldatapb.Ch return nil, err } + span.Annotate("before_tablet_type", topoproto.TabletTypeLString(tablet.Type)) + if !topo.IsTrivialTypeChange(tablet.Type, req.DbType) { return nil, fmt.Errorf("tablet %v type change %v -> %v is not an allowed transition for ChangeTabletType", req.TabletAlias, tablet.Type, req.DbType) } @@ -237,6 +266,16 @@ func (s *VtctldServer) ChangeTabletType(ctx context.Context, req *vtctldatapb.Ch // CreateKeyspace is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) CreateKeyspace(ctx context.Context, req *vtctldatapb.CreateKeyspaceRequest) (*vtctldatapb.CreateKeyspaceResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.CreateKeyspace") + defer span.Finish() + + span.Annotate("keyspace", req.Name) + span.Annotate("keyspace_type", topoproto.KeyspaceTypeLString(req.Type)) + span.Annotate("sharding_column_name", req.ShardingColumnName) + span.Annotate("sharding_column_type", topoproto.KeyspaceIDTypeLString(req.ShardingColumnType)) + span.Annotate("force", req.Force) + span.Annotate("allow_empty_vschema", req.AllowEmptyVSchema) + switch req.Type { case topodatapb.KeyspaceType_NORMAL: case topodatapb.KeyspaceType_SNAPSHOT: @@ -247,6 +286,9 @@ func (s *VtctldServer) CreateKeyspace(ctx context.Context, req *vtctldatapb.Crea if req.SnapshotTime == nil { return nil, errors.New("SnapshotTime is required for SNAPSHOT keyspaces") } + + span.Annotate("base_keyspace", req.BaseKeyspace) + span.Annotate("snapshot_time", req.SnapshotTime) // TODO: get a proper string repr default: return nil, fmt.Errorf("unknown keyspace type %v", req.Type) } @@ -325,6 +367,14 @@ func (s *VtctldServer) CreateKeyspace(ctx context.Context, req *vtctldatapb.Crea // CreateShard is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) CreateShard(ctx context.Context, req *vtctldatapb.CreateShardRequest) (*vtctldatapb.CreateShardResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.CreateShard") + defer span.Finish() + + span.Annotate("keyspace", req.Keyspace) + span.Annotate("shard", req.ShardName) + span.Annotate("force", req.Force) + span.Annotate("include_parent", req.IncludeParent) + if req.IncludeParent { log.Infof("Creating empty keyspace for %s", req.Keyspace) if err := s.ts.CreateKeyspace(ctx, req.Keyspace, &topodatapb.Keyspace{}); err != nil { @@ -376,6 +426,12 @@ func (s *VtctldServer) CreateShard(ctx context.Context, req *vtctldatapb.CreateS // DeleteCellInfo is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) DeleteCellInfo(ctx context.Context, req *vtctldatapb.DeleteCellInfoRequest) (*vtctldatapb.DeleteCellInfoResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.DeleteCellInfo") + defer span.Finish() + + span.Annotate("cell", req.Name) + span.Annotate("force", req.Force) + ctx, cancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout) defer cancel() @@ -388,6 +444,11 @@ func (s *VtctldServer) DeleteCellInfo(ctx context.Context, req *vtctldatapb.Dele // DeleteCellsAlias is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) DeleteCellsAlias(ctx context.Context, req *vtctldatapb.DeleteCellsAliasRequest) (*vtctldatapb.DeleteCellsAliasResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.DeleteCellsAlias") + defer span.Finish() + + span.Annotate("cells_alias", req.Name) + ctx, cancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout) defer cancel() @@ -400,6 +461,12 @@ func (s *VtctldServer) DeleteCellsAlias(ctx context.Context, req *vtctldatapb.De // DeleteKeyspace is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) DeleteKeyspace(ctx context.Context, req *vtctldatapb.DeleteKeyspaceRequest) (*vtctldatapb.DeleteKeyspaceResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.DeleteKeyspace") + defer span.Finish() + + span.Annotate("keyspace", req.Keyspace) + span.Annotate("recursive", req.Recursive) + shards, err := s.ts.GetShardNames(ctx, req.Keyspace) if err != nil { return nil, err @@ -446,6 +513,9 @@ func (s *VtctldServer) DeleteKeyspace(ctx context.Context, req *vtctldatapb.Dele // DeleteShards is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) DeleteShards(ctx context.Context, req *vtctldatapb.DeleteShardsRequest) (*vtctldatapb.DeleteShardsResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.DeleteShards") + defer span.Finish() + for _, shard := range req.Shards { if err := deleteShard(ctx, s.ts, shard.Keyspace, shard.Name, req.Recursive, req.EvenIfServing); err != nil { return nil, err @@ -457,6 +527,9 @@ func (s *VtctldServer) DeleteShards(ctx context.Context, req *vtctldatapb.Delete // DeleteTablets is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) DeleteTablets(ctx context.Context, req *vtctldatapb.DeleteTabletsRequest) (*vtctldatapb.DeleteTabletsResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.DeleteTablets") + defer span.Finish() + for _, alias := range req.TabletAliases { if err := deleteTablet(ctx, s.ts, alias, req.AllowPrimary); err != nil { return nil, err diff --git a/go/vt/vtctl/grpcvtctldserver/topo.go b/go/vt/vtctl/grpcvtctldserver/topo.go index 7de161bcd2..14c6944182 100644 --- a/go/vt/vtctl/grpcvtctldserver/topo.go +++ b/go/vt/vtctl/grpcvtctldserver/topo.go @@ -20,6 +20,7 @@ import ( "context" "fmt" + "vitess.io/vitess/go/trace" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/topo/topoproto" @@ -31,6 +32,14 @@ import ( ) func deleteShard(ctx context.Context, ts *topo.Server, keyspace string, shard string, recursive bool, evenIfServing bool) error { + span, ctx := trace.NewSpan(ctx, "VtctldServer.deleteShard") + defer span.Finish() + + span.Annotate("keyspace", keyspace) + span.Annotate("shard", shard) + span.Annotate("recursive", recursive) + span.Annotate("even_if_serving", evenIfServing) + // Read the Shard object. If it's not in the topo, try to clean up the topo // anyway. shardInfo, err := ts.GetShard(ctx, keyspace, shard) From f649cb696eef8f3cd080646132af15f8ad478ba2 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Sun, 6 Jun 2021 13:33:03 -0400 Subject: [PATCH 39/64] Fix typo in comment Signed-off-by: Andrew Mason --- go/vt/vtctl/grpcvtctldserver/topo.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/vtctl/grpcvtctldserver/topo.go b/go/vt/vtctl/grpcvtctldserver/topo.go index 14c6944182..4363a0c217 100644 --- a/go/vt/vtctl/grpcvtctldserver/topo.go +++ b/go/vt/vtctl/grpcvtctldserver/topo.go @@ -243,7 +243,7 @@ func removeShardCell(ctx context.Context, ts *topo.Server, cell string, keyspace if recursive { log.Infof("Deleting all tablets in cell %v in shard %v/%v", cell, keyspace, shardName) for _, node := range replication.Nodes { - // We don't care about scraping our updating the replication + // We don't care about scrapping or updating the replication // graph, because we're about to delete the entire replication // graph. log.Infof("Deleting tablet %v", topoproto.TabletAliasString(node.TabletAlias)) From 100e294b6314b8be8b84b744c33dc5729c65174a Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Sun, 6 Jun 2021 13:37:18 -0400 Subject: [PATCH 40/64] fixup! [vtctldserver] wip -- start tracing all vtctldserver endpoints Signed-off-by: Andrew Mason --- go/vt/vtctl/grpcvtctldserver/server.go | 195 ++++++++++++++++++++++++- go/vt/vtctl/grpcvtctldserver/topo.go | 25 ++++ 2 files changed, 217 insertions(+), 3 deletions(-) diff --git a/go/vt/vtctl/grpcvtctldserver/server.go b/go/vt/vtctl/grpcvtctldserver/server.go index 40ef1997fd..122cc1dcec 100644 --- a/go/vt/vtctl/grpcvtctldserver/server.go +++ b/go/vt/vtctl/grpcvtctldserver/server.go @@ -516,6 +516,10 @@ func (s *VtctldServer) DeleteShards(ctx context.Context, req *vtctldatapb.Delete span, ctx := trace.NewSpan(ctx, "VtctldServer.DeleteShards") defer span.Finish() + span.Annotate("num_shards", len(req.Shards)) + span.Annotate("even_if_serving", req.EvenIfServing) + span.Annotate("recursive", req.Recursive) + for _, shard := range req.Shards { if err := deleteShard(ctx, s.ts, shard.Keyspace, shard.Name, req.Recursive, req.EvenIfServing); err != nil { return nil, err @@ -530,6 +534,9 @@ func (s *VtctldServer) DeleteTablets(ctx context.Context, req *vtctldatapb.Delet span, ctx := trace.NewSpan(ctx, "VtctldServer.DeleteTablets") defer span.Finish() + span.Annotate("num_tablets", len(req.TabletAliases)) + span.Annotate("allow_primary", req.AllowPrimary) + for _, alias := range req.TabletAliases { if err := deleteTablet(ctx, s.ts, alias, req.AllowPrimary); err != nil { return nil, err @@ -541,6 +548,16 @@ func (s *VtctldServer) DeleteTablets(ctx context.Context, req *vtctldatapb.Delet // EmergencyReparentShard is part of the vtctldservicepb.VtctldServer interface. func (s *VtctldServer) EmergencyReparentShard(ctx context.Context, req *vtctldatapb.EmergencyReparentShardRequest) (*vtctldatapb.EmergencyReparentShardResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.EmergencyReparentShard") + defer span.Finish() + + span.Annotate("keyspace", req.Keyspace) + span.Annotate("shard", req.Shard) + span.Annotate("new_primary_alias", topoproto.TabletAliasString(req.NewPrimary)) + + ignoreReplicaAliases := topoproto.TabletAliasList(req.IgnoreReplicas).ToStringSlice() + span.Annotate("ignore_replicas", strings.Join(ignoreReplicaAliases, ",")) + waitReplicasTimeout, ok, err := protoutil.DurationFromProto(req.WaitReplicasTimeout) if err != nil { return nil, err @@ -548,6 +565,8 @@ func (s *VtctldServer) EmergencyReparentShard(ctx context.Context, req *vtctldat waitReplicasTimeout = time.Second * 30 } + span.Annotate("wait_replicas_timeout_sec", waitReplicasTimeout.Seconds()) + m := sync.RWMutex{} logstream := []*logutilpb.Event{} logger := logutil.NewCallbackLogger(func(e *logutilpb.Event) { @@ -562,7 +581,7 @@ func (s *VtctldServer) EmergencyReparentShard(ctx context.Context, req *vtctldat req.Shard, reparentutil.EmergencyReparentOptions{ NewPrimaryAlias: req.NewPrimary, - IgnoreReplicas: sets.NewString(topoproto.TabletAliasList(req.IgnoreReplicas).ToStringSlice()...), + IgnoreReplicas: sets.NewString(ignoreReplicaAliases...), WaitReplicasTimeout: waitReplicasTimeout, }, ) @@ -592,6 +611,11 @@ func (s *VtctldServer) EmergencyReparentShard(ctx context.Context, req *vtctldat // FindAllShardsInKeyspace is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) FindAllShardsInKeyspace(ctx context.Context, req *vtctldatapb.FindAllShardsInKeyspaceRequest) (*vtctldatapb.FindAllShardsInKeyspaceResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.FindAllShardsInKeyspace") + defer span.Finish() + + span.Annotate("keyspace", req.Keyspace) + result, err := s.ts.FindAllShardsInKeyspace(ctx, req.Keyspace) if err != nil { return nil, err @@ -613,14 +637,21 @@ func (s *VtctldServer) FindAllShardsInKeyspace(ctx context.Context, req *vtctlda // GetBackups is part of the vtctldservicepb.VtctldServer interface. func (s *VtctldServer) GetBackups(ctx context.Context, req *vtctldatapb.GetBackupsRequest) (*vtctldatapb.GetBackupsResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.GetBackups") + defer span.Finish() + + span.Annotate("keyspace", req.Keyspace) + span.Annotate("shard", req.Shard) + bs, err := backupstorage.GetBackupStorage() if err != nil { return nil, err } - defer bs.Close() bucket := filepath.Join(req.Keyspace, req.Shard) + span.Annotate("backup_path", bucket) + bhs, err := bs.ListBackups(ctx, bucket) if err != nil { return nil, err @@ -639,6 +670,9 @@ func (s *VtctldServer) GetBackups(ctx context.Context, req *vtctldatapb.GetBacku // GetCellInfoNames is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) GetCellInfoNames(ctx context.Context, req *vtctldatapb.GetCellInfoNamesRequest) (*vtctldatapb.GetCellInfoNamesResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.GetCellInfoNames") + defer span.Finish() + names, err := s.ts.GetCellInfoNames(ctx) if err != nil { return nil, err @@ -649,10 +683,15 @@ func (s *VtctldServer) GetCellInfoNames(ctx context.Context, req *vtctldatapb.Ge // GetCellInfo is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) GetCellInfo(ctx context.Context, req *vtctldatapb.GetCellInfoRequest) (*vtctldatapb.GetCellInfoResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.GetCellInfo") + defer span.Finish() + if req.Cell == "" { return nil, vterrors.Errorf(vtrpc.Code_INVALID_ARGUMENT, "cell field is required") } + span.Annotate("cell", req.Cell) + // We use a strong read, because users using this command want the latest // data, and this is user-generated, not used in any automated process. strongRead := true @@ -666,6 +705,9 @@ func (s *VtctldServer) GetCellInfo(ctx context.Context, req *vtctldatapb.GetCell // GetCellsAliases is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) GetCellsAliases(ctx context.Context, req *vtctldatapb.GetCellsAliasesRequest) (*vtctldatapb.GetCellsAliasesResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.GetCellsAliases") + defer span.Finish() + strongRead := true aliases, err := s.ts.GetCellsAliases(ctx, strongRead) if err != nil { @@ -677,6 +719,11 @@ func (s *VtctldServer) GetCellsAliases(ctx context.Context, req *vtctldatapb.Get // GetKeyspace is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) GetKeyspace(ctx context.Context, req *vtctldatapb.GetKeyspaceRequest) (*vtctldatapb.GetKeyspaceResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.GetKeyspace") + defer span.Finish() + + span.Annotate("keyspace", req.Keyspace) + keyspace, err := s.ts.GetKeyspace(ctx, req.Keyspace) if err != nil { return nil, err @@ -692,6 +739,9 @@ func (s *VtctldServer) GetKeyspace(ctx context.Context, req *vtctldatapb.GetKeys // GetKeyspaces is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) GetKeyspaces(ctx context.Context, req *vtctldatapb.GetKeyspacesRequest) (*vtctldatapb.GetKeyspacesResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.GetKeyspaces") + defer span.Finish() + names, err := s.ts.GetKeyspaces(ctx) if err != nil { return nil, err @@ -713,6 +763,9 @@ func (s *VtctldServer) GetKeyspaces(ctx context.Context, req *vtctldatapb.GetKey // GetRoutingRules is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) GetRoutingRules(ctx context.Context, req *vtctldatapb.GetRoutingRulesRequest) (*vtctldatapb.GetRoutingRulesResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.GetRoutingRules") + defer span.Finish() + rr, err := s.ts.GetRoutingRules(ctx) if err != nil { return nil, err @@ -725,11 +778,22 @@ func (s *VtctldServer) GetRoutingRules(ctx context.Context, req *vtctldatapb.Get // GetSchema is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) GetSchema(ctx context.Context, req *vtctldatapb.GetSchemaRequest) (*vtctldatapb.GetSchemaResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.GetSchema") + defer span.Finish() + + span.Annotate("tablet_alias", topoproto.TabletAliasString(req.TabletAlias)) + tablet, err := s.ts.GetTablet(ctx, req.TabletAlias) if err != nil { return nil, fmt.Errorf("GetTablet(%v) failed: %w", req.TabletAlias, err) } + span.Annotate("tables", strings.Join(req.Tables, ",")) + span.Annotate("exclude_tables", strings.Join(req.ExcludeTables, ",")) + span.Annotate("include_views", req.IncludeViews) + span.Annotate("table_names_only", req.TableNamesOnly) + span.Annotate("table_sizes_only", req.TableSizesOnly) + sd, err := s.tmc.GetSchema(ctx, tablet.Tablet, req.Tables, req.ExcludeTables, req.IncludeViews) if err != nil { return nil, fmt.Errorf("GetSchema(%v, %v, %v, %v) failed: %w", tablet.Tablet, req.Tables, req.ExcludeTables, req.IncludeViews, err) @@ -767,6 +831,12 @@ func (s *VtctldServer) GetSchema(ctx context.Context, req *vtctldatapb.GetSchema // GetShard is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) GetShard(ctx context.Context, req *vtctldatapb.GetShardRequest) (*vtctldatapb.GetShardResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.GetShard") + defer span.Finish() + + span.Annotate("keyspace", req.Keyspace) + span.Annotate("shard", req.ShardName) + shard, err := s.ts.GetShard(ctx, req.Keyspace, req.ShardName) if err != nil { return nil, err @@ -783,6 +853,9 @@ func (s *VtctldServer) GetShard(ctx context.Context, req *vtctldatapb.GetShardRe // GetSrvKeyspaces is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) GetSrvKeyspaces(ctx context.Context, req *vtctldatapb.GetSrvKeyspacesRequest) (*vtctldatapb.GetSrvKeyspacesResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.GetSrvKeyspaces") + defer span.Finish() + cells := req.Cells if len(cells) == 0 { @@ -794,6 +867,8 @@ func (s *VtctldServer) GetSrvKeyspaces(ctx context.Context, req *vtctldatapb.Get } } + span.Annotate("cells", strings.Join(cells, ",")) + srvKeyspaces := make(map[string]*topodatapb.SrvKeyspace, len(cells)) for _, cell := range cells { @@ -819,6 +894,11 @@ func (s *VtctldServer) GetSrvKeyspaces(ctx context.Context, req *vtctldatapb.Get // GetSrvVSchema is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) GetSrvVSchema(ctx context.Context, req *vtctldatapb.GetSrvVSchemaRequest) (*vtctldatapb.GetSrvVSchemaResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.GetSrvVSchema") + defer span.Finish() + + span.Annotate("cell", req.Cell) + vschema, err := s.ts.GetSrvVSchema(ctx, req.Cell) if err != nil { return nil, err @@ -831,6 +911,9 @@ func (s *VtctldServer) GetSrvVSchema(ctx context.Context, req *vtctldatapb.GetSr // GetSrvVSchemas is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) GetSrvVSchemas(ctx context.Context, req *vtctldatapb.GetSrvVSchemasRequest) (*vtctldatapb.GetSrvVSchemasResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.GetSrvVSchemas") + defer span.Finish() + allCells, err := s.ts.GetCellInfoNames(ctx) if err != nil { return nil, err @@ -846,6 +929,7 @@ func (s *VtctldServer) GetSrvVSchemas(ctx context.Context, req *vtctldatapb.GetS cells = s1.Intersection(s2).List() } + span.Annotate("cells", strings.Join(cells, ",")) svs := make(map[string]*vschemapb.SrvVSchema, len(cells)) for _, cell := range cells { @@ -870,6 +954,11 @@ func (s *VtctldServer) GetSrvVSchemas(ctx context.Context, req *vtctldatapb.GetS // GetTablet is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) GetTablet(ctx context.Context, req *vtctldatapb.GetTabletRequest) (*vtctldatapb.GetTabletResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.GetTablet") + defer span.Finish() + + span.Annotate("tablet_alias", topoproto.TabletAliasString(req.TabletAlias)) + ti, err := s.ts.GetTablet(ctx, req.TabletAlias) if err != nil { return nil, err @@ -882,6 +971,12 @@ func (s *VtctldServer) GetTablet(ctx context.Context, req *vtctldatapb.GetTablet // GetTablets is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) GetTablets(ctx context.Context, req *vtctldatapb.GetTabletsRequest) (*vtctldatapb.GetTabletsResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.GetTablets") + defer span.Finish() + + span.Annotate("cells", strings.Join(req.Cells, ",")) + span.Annotate("strict", req.Strict) + // It is possible that an old primary has not yet updated its type in the // topo. In that case, report its type as UNKNOWN. It used to be MASTER but // is no longer the serving primary. @@ -906,11 +1001,16 @@ func (s *VtctldServer) GetTablets(ctx context.Context, req *vtctldatapb.GetTable switch { case len(req.TabletAliases) > 0: + span.Annotate("tablet_aliases", strings.Join(topoproto.TabletAliasList(req.TabletAliases).ToStringSlice(), ",")) + tabletMap, err = s.ts.GetTabletMap(ctx, req.TabletAliases) if err != nil { err = fmt.Errorf("GetTabletMap(%v) failed: %w", req.TabletAliases, err) } case req.Keyspace != "" && req.Shard != "": + span.Annotate("keyspace", req.Keyspace) + span.Annotate("shard", req.Shard) + tabletMap, err = s.ts.GetTabletMapForShard(ctx, req.Keyspace, req.Shard) if err != nil { err = fmt.Errorf("GetTabletMapForShard(%s, %s) failed: %w", req.Keyspace, req.Shard, err) @@ -1043,6 +1143,11 @@ func (s *VtctldServer) GetTablets(ctx context.Context, req *vtctldatapb.GetTable // GetVSchema is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) GetVSchema(ctx context.Context, req *vtctldatapb.GetVSchemaRequest) (*vtctldatapb.GetVSchemaResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.GetVSchema") + defer span.Finish() + + span.Annotate("keyspace", req.Keyspace) + vschema, err := s.ts.GetVSchema(ctx, req.Keyspace) if err != nil { return nil, err @@ -1066,6 +1171,9 @@ func (s *VtctldServer) GetWorkflows(ctx context.Context, req *vtctldatapb.GetWor // InitShardPrimary is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) InitShardPrimary(ctx context.Context, req *vtctldatapb.InitShardPrimaryRequest) (*vtctldatapb.InitShardPrimaryResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.InitShardPrimary") + defer span.Finish() + if req.Keyspace == "" { return nil, vterrors.Errorf(vtrpc.Code_INVALID_ARGUMENT, "keyspace field is required") } @@ -1081,6 +1189,11 @@ func (s *VtctldServer) InitShardPrimary(ctx context.Context, req *vtctldatapb.In waitReplicasTimeout = time.Second * 30 } + span.Annotate("keyspace", req.Keyspace) + span.Annotate("shard", req.Shard) + span.Annotate("wait_replicas_timeout_sec", waitReplicasTimeout.Seconds()) + span.Annotate("force", req.Force) + ctx, unlock, err := s.ts.LockShard(ctx, req.Keyspace, req.Shard, fmt.Sprintf("InitShardPrimary(%v)", topoproto.TabletAliasString(req.PrimaryElectTabletAlias))) if err != nil { return nil, err @@ -1092,7 +1205,7 @@ func (s *VtctldServer) InitShardPrimary(ctx context.Context, req *vtctldatapb.In logstream := []*logutilpb.Event{} resp := &vtctldatapb.InitShardPrimaryResponse{} - err = s.InitShardPrimaryLocked(ctx, ev, req, waitReplicasTimeout, tmclient.NewTabletManagerClient(), logutil.NewCallbackLogger(func(e *logutilpb.Event) { + err = s.InitShardPrimaryLocked(ctx, ev, req, waitReplicasTimeout, s.tmc, logutil.NewCallbackLogger(func(e *logutilpb.Event) { m.Lock() defer m.Unlock() @@ -1318,6 +1431,9 @@ func (s *VtctldServer) InitShardPrimaryLocked( // PlannedReparentShard is part of the vtctldservicepb.VtctldServer interface. func (s *VtctldServer) PlannedReparentShard(ctx context.Context, req *vtctldatapb.PlannedReparentShardRequest) (*vtctldatapb.PlannedReparentShardResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.PlannedReparentShard") + defer span.Finish() + waitReplicasTimeout, ok, err := protoutil.DurationFromProto(req.WaitReplicasTimeout) if err != nil { return nil, err @@ -1325,6 +1441,18 @@ func (s *VtctldServer) PlannedReparentShard(ctx context.Context, req *vtctldatap waitReplicasTimeout = time.Second * 30 } + span.Annotate("keyspace", req.Keyspace) + span.Annotate("shard", req.Shard) + span.Annotate("wait_replicas_timeout_sec", waitReplicasTimeout.Seconds()) + + if req.AvoidPrimary != nil { + span.Annotate("avoid_primary_alias", topoproto.TabletAliasString(req.AvoidPrimary)) + } + + if req.NewPrimary != nil { + span.Annotate("new_primary_alias", topoproto.TabletAliasString(req.NewPrimary)) + } + m := sync.RWMutex{} logstream := []*logutilpb.Event{} logger := logutil.NewCallbackLogger(func(e *logutilpb.Event) { @@ -1369,6 +1497,11 @@ func (s *VtctldServer) PlannedReparentShard(ctx context.Context, req *vtctldatap // RebuildVSchemaGraph is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) RebuildVSchemaGraph(ctx context.Context, req *vtctldatapb.RebuildVSchemaGraphRequest) (*vtctldatapb.RebuildVSchemaGraphResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.RebuildVSchemaGraph") + defer span.Finish() + + span.Annotate("cells", strings.Join(req.Cells, ",")) + if err := s.ts.RebuildSrvVSchema(ctx, req.Cells); err != nil { return nil, err } @@ -1378,6 +1511,14 @@ func (s *VtctldServer) RebuildVSchemaGraph(ctx context.Context, req *vtctldatapb // RemoveKeyspaceCell is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) RemoveKeyspaceCell(ctx context.Context, req *vtctldatapb.RemoveKeyspaceCellRequest) (*vtctldatapb.RemoveKeyspaceCellResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.RemoveKeyspaceCell") + defer span.Finish() + + span.Annotate("keyspace", req.Keyspace) + span.Annotate("cell", req.Cell) + span.Annotate("force", req.Force) + span.Annotate("recursive", req.Recursive) + shards, err := s.ts.GetShardNames(ctx, req.Keyspace) if err != nil { return nil, err @@ -1402,6 +1543,15 @@ func (s *VtctldServer) RemoveKeyspaceCell(ctx context.Context, req *vtctldatapb. // RemoveShardCell is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) RemoveShardCell(ctx context.Context, req *vtctldatapb.RemoveShardCellRequest) (*vtctldatapb.RemoveShardCellResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.RemoveShardCell") + defer span.Finish() + + span.Annotate("keyspace", req.Keyspace) + span.Annotate("shard", req.ShardName) + span.Annotate("cell", req.Cell) + span.Annotate("force", req.Force) + span.Annotate("recursive", req.Recursive) + if err := removeShardCell(ctx, s.ts, req.Cell, req.Keyspace, req.ShardName, req.Recursive, req.Force); err != nil { return nil, err } @@ -1411,10 +1561,15 @@ func (s *VtctldServer) RemoveShardCell(ctx context.Context, req *vtctldatapb.Rem // ReparentTablet is part of the vtctldservicepb.VtctldServer interface. func (s *VtctldServer) ReparentTablet(ctx context.Context, req *vtctldatapb.ReparentTabletRequest) (*vtctldatapb.ReparentTabletResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.ReparentTablet") + defer span.Finish() + if req.Tablet == nil { return nil, vterrors.Errorf(vtrpc.Code_INVALID_ARGUMENT, "tablet alias must not be nil") } + span.Annotate("tablet_alias", topoproto.TabletAliasString(req.Tablet)) + tablet, err := s.ts.GetTablet(ctx, req.Tablet) if err != nil { return nil, err @@ -1459,6 +1614,12 @@ func (s *VtctldServer) ReparentTablet(ctx context.Context, req *vtctldatapb.Repa // ShardReplicationPositions is part of the vtctldservicepb.VtctldServer interface. func (s *VtctldServer) ShardReplicationPositions(ctx context.Context, req *vtctldatapb.ShardReplicationPositionsRequest) (*vtctldatapb.ShardReplicationPositionsResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.ShardReplicationPositions") + defer span.Finish() + + span.Annotate("keyspace", req.Keyspace) + span.Annotate("shard", req.Shard) + tabletInfoMap, err := s.ts.GetTabletMapForShard(ctx, req.Keyspace, req.Shard) if err != nil { return nil, fmt.Errorf("GetTabletMapForShard(%s, %s) failed: %w", req.Keyspace, req.Shard, err) @@ -1490,6 +1651,11 @@ func (s *VtctldServer) ShardReplicationPositions(ctx context.Context, req *vtctl go func(ctx context.Context, alias string, tablet *topodatapb.Tablet) { defer wg.Done() + span, ctx := trace.NewSpan(ctx, "VtctldServer.getPrimaryPosition") + defer span.Finish() + + span.Annotate("tablet_alias", alias) + ctx, cancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout) defer cancel() @@ -1527,6 +1693,11 @@ func (s *VtctldServer) ShardReplicationPositions(ctx context.Context, req *vtctl go func(ctx context.Context, alias string, tablet *topodatapb.Tablet) { defer wg.Done() + span, ctx := trace.NewSpan(ctx, "VtctldServer.getReplicationStatus") + defer span.Finish() + + span.Annotate("tablet_alias", alias) + ctx, cancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout) defer cancel() @@ -1570,10 +1741,15 @@ func (s *VtctldServer) ShardReplicationPositions(ctx context.Context, req *vtctl // TabletExternallyReparented is part of the vtctldservicepb.VtctldServer interface. func (s *VtctldServer) TabletExternallyReparented(ctx context.Context, req *vtctldatapb.TabletExternallyReparentedRequest) (*vtctldatapb.TabletExternallyReparentedResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.TabletExternallyReparented") + defer span.Finish() + if req.Tablet == nil { return nil, vterrors.Errorf(vtrpc.Code_INVALID_ARGUMENT, "TabletExternallyReparentedRequest.Tablet must not be nil") } + span.Annotate("tablet_alias", topoproto.TabletAliasString(req.Tablet)) + tablet, err := s.ts.GetTablet(ctx, req.Tablet) if err != nil { log.Warningf("TabletExternallyReparented: failed to read tablet record for %v: %v", topoproto.TabletAliasString(req.Tablet), err) @@ -1630,6 +1806,13 @@ func (s *VtctldServer) TabletExternallyReparented(ctx context.Context, req *vtct // UpdateCellInfo is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) UpdateCellInfo(ctx context.Context, req *vtctldatapb.UpdateCellInfoRequest) (*vtctldatapb.UpdateCellInfoResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.UpdateCellInfo") + defer span.Finish() + + span.Annotate("cell", req.Name) + span.Annotate("cell_server_address", req.CellInfo.ServerAddress) + span.Annotate("cell_root", req.CellInfo.Root) + ctx, cancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout) defer cancel() @@ -1670,6 +1853,12 @@ func (s *VtctldServer) UpdateCellInfo(ctx context.Context, req *vtctldatapb.Upda // UpdateCellsAlias is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) UpdateCellsAlias(ctx context.Context, req *vtctldatapb.UpdateCellsAliasRequest) (*vtctldatapb.UpdateCellsAliasResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.UpdateCellsAlias") + defer span.Finish() + + span.Annotate("cells_alias", req.Name) + span.Annotate("cells_alias_cells", strings.Join(req.CellsAlias.Cells, ",")) + ctx, cancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout) defer cancel() diff --git a/go/vt/vtctl/grpcvtctldserver/topo.go b/go/vt/vtctl/grpcvtctldserver/topo.go index 4363a0c217..a8d501d152 100644 --- a/go/vt/vtctl/grpcvtctldserver/topo.go +++ b/go/vt/vtctl/grpcvtctldserver/topo.go @@ -90,6 +90,14 @@ func deleteShard(ctx context.Context, ts *topo.Server, keyspace string, shard st // distinct from the RemoveShardCell rpc. Despite having similar names, they are // **not** the same! func deleteShardCell(ctx context.Context, ts *topo.Server, keyspace string, shard string, cell string, recursive bool) error { + span, ctx := trace.NewSpan(ctx, "VtctldServer.deleteShardCell") + defer span.Finish() + + span.Annotate("keyspace", keyspace) + span.Annotate("shard", shard) + span.Annotate("cell", cell) + span.Annotate("recursive", recursive) + var aliases []*topodatapb.TabletAlias // Get the ShardReplication object for the cell. Collect all the tablets @@ -165,6 +173,12 @@ func deleteShardCell(ctx context.Context, ts *topo.Server, keyspace string, shar } func deleteTablet(ctx context.Context, ts *topo.Server, alias *topodatapb.TabletAlias, allowPrimary bool) (err error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.deleteTablet") + defer span.Finish() + + span.Annotate("tablet_alias", topoproto.TabletAliasString(alias)) + span.Annotate("allow_primary", allowPrimary) + tablet, err := ts.GetTablet(ctx, alias) if err != nil { return err @@ -175,6 +189,8 @@ func deleteTablet(ctx context.Context, ts *topo.Server, alias *topodatapb.Tablet return err } + span.Annotate("is_primary", isPrimary) + if isPrimary && !allowPrimary { return vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "cannot delete tablet %v as it is a master, pass AllowPrimary = true", topoproto.TabletAliasString(alias)) } @@ -218,6 +234,15 @@ func deleteTablet(ctx context.Context, ts *topo.Server, alias *topodatapb.Tablet } func removeShardCell(ctx context.Context, ts *topo.Server, cell string, keyspace string, shardName string, recursive bool, force bool) error { + span, ctx := trace.NewSpan(ctx, "VtctldServer.removeShardCell") + defer span.Finish() + + span.Annotate("keyspace", keyspace) + span.Annotate("shard", shardName) + span.Annotate("cell", cell) + span.Annotate("recursive", recursive) + span.Annotate("force", force) + shard, err := ts.GetShard(ctx, keyspace, shardName) if err != nil { return err From a5633cc980c63f057a25000db3f2ecbee051cad3 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 8 Jun 2021 10:46:31 -0400 Subject: [PATCH 41/64] [vtctldserver] Update `ApplyVSchema` with tracing after rebase Signed-off-by: Andrew Mason --- go/vt/vtctl/grpcvtctldserver/server.go | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/go/vt/vtctl/grpcvtctldserver/server.go b/go/vt/vtctl/grpcvtctldserver/server.go index 122cc1dcec..8ce8e329d3 100644 --- a/go/vt/vtctl/grpcvtctldserver/server.go +++ b/go/vt/vtctl/grpcvtctldserver/server.go @@ -25,9 +25,8 @@ import ( "sync" "time" - "google.golang.org/protobuf/proto" - "google.golang.org/grpc" + "google.golang.org/protobuf/proto" "k8s.io/apimachinery/pkg/util/sets" "vitess.io/vitess/go/event" @@ -152,6 +151,14 @@ func (s *VtctldServer) ApplyRoutingRules(ctx context.Context, req *vtctldatapb.A // ApplyVSchema is part of the vtctlservicepb.VtctldServer interface. func (s *VtctldServer) ApplyVSchema(ctx context.Context, req *vtctldatapb.ApplyVSchemaRequest) (*vtctldatapb.ApplyVSchemaResponse, error) { + span, ctx := trace.NewSpan(ctx, "VtctldServer.ApplyVSchema") + defer span.Finish() + + span.Annotate("keyspace", req.Keyspace) + span.Annotate("cells", strings.Join(req.Cells, ",")) + span.Annotate("skip_rebuild", req.SkipRebuild) + span.Annotate("dry_run", req.DryRun) + if _, err := s.ts.GetKeyspace(ctx, req.Keyspace); err != nil { if topo.IsErrType(err, topo.NoNode) { return nil, vterrors.Wrapf(err, "keyspace(%s) doesn't exist, check if the keyspace is initialized", req.Keyspace) @@ -163,10 +170,14 @@ func (s *VtctldServer) ApplyVSchema(ctx context.Context, req *vtctldatapb.ApplyV return nil, vterrors.New(vtrpc.Code_INVALID_ARGUMENT, "must pass exactly one of req.VSchema and req.Sql") } - var vs *vschemapb.Keyspace - var err error + var ( + vs *vschemapb.Keyspace + err error + ) if req.Sql != "" { + span.Annotate("sql_mode", true) + stmt, err := sqlparser.Parse(req.Sql) if err != nil { return nil, vterrors.Wrapf(err, "Parse(%s)", req.Sql) @@ -186,6 +197,7 @@ func (s *VtctldServer) ApplyVSchema(ctx context.Context, req *vtctldatapb.ApplyV return nil, vterrors.Wrapf(err, "ApplyVSchemaDDL(%s,%v,%v)", req.Keyspace, vs, ddl) } } else { // "jsonMode" + span.Annotate("sql_mode", false) vs = req.VSchema } From 6b52e982ccdf1c20b1b11d84693e5d57da6b46b6 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Tue, 8 Jun 2021 17:20:51 +0200 Subject: [PATCH 42/64] Address review comments Signed-off-by: Rohit Nayak --- .../tabletmanager/vreplication/table_plan_builder.go | 6 ++---- go/vt/vttablet/tabletmanager/vreplication/vreplicator.go | 6 +++--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/go/vt/vttablet/tabletmanager/vreplication/table_plan_builder.go b/go/vt/vttablet/tabletmanager/vreplication/table_plan_builder.go index 38b69f1060..9d094b7d46 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/table_plan_builder.go +++ b/go/vt/vttablet/tabletmanager/vreplication/table_plan_builder.go @@ -772,14 +772,12 @@ func (tpb *tablePlanBuilder) generatePKConstraint(buf *sqlparser.TrackedBuffer, } func (tpb *tablePlanBuilder) isColumnGenerated(col sqlparser.ColIdent) bool { - isGenerated := false for _, colInfo := range tpb.colInfos { if col.EqualString(colInfo.Name) && colInfo.IsGenerated { - isGenerated = true - break + return true } } - return isGenerated + return false } // bindvarFormatter is a dual mode formatter. Its behavior diff --git a/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go b/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go index cd827ba26e..2c12568f52 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go @@ -224,7 +224,7 @@ func (vr *vreplicator) replicate(ctx context.Context) error { } } -// ColumnInfo is used to store charset and collation for primary keys where applicable +// ColumnInfo is used to store charset and collation type ColumnInfo struct { Name string CharSet string @@ -292,8 +292,8 @@ func (vr *vreplicator) buildColInfoMap(ctx context.Context) (map[string][]*Colum isPK = true } } - extra := row[5].ToString() - if strings.Contains(strings.ToLower(extra), "generated") { + extra := strings.ToLower(row[5].ToString()) + if strings.Contains(extra, "generated") || strings.Contains(extra, "virtual") { isGenerated = true } colInfo = append(colInfo, &ColumnInfo{ From 3f0d6bfceb7700ac1a517542c4355cb65b160fa7 Mon Sep 17 00:00:00 2001 From: Rafael Chacon Date: Tue, 8 Jun 2021 14:37:34 -0700 Subject: [PATCH 43/64] Adds padding when doing keyrange comparisons Signed-off-by: Rafael Chacon --- go/vt/key/key.go | 26 +++++- go/vt/key/key_test.go | 161 ++++++++++++++++++++++++++++++++++ go/vt/topotools/split_test.go | 12 +++ 3 files changed, 195 insertions(+), 4 deletions(-) diff --git a/go/vt/key/key.go b/go/vt/key/key.go index ed48825783..e271886012 100644 --- a/go/vt/key/key.go +++ b/go/vt/key/key.go @@ -194,8 +194,26 @@ func KeyRangeEqual(left, right *topodatapb.KeyRange) bool { if right == nil { return len(left.Start) == 0 && len(left.End) == 0 } - return bytes.Equal(left.Start, right.Start) && - bytes.Equal(left.End, right.End) + return bytes.Equal(addPadding(left.Start), addPadding(right.Start)) && + bytes.Equal(addPadding(left.End), addPadding(right.End)) +} + +// addPadding adds padding to make sure keyrange represents an 8 byte integer. +// From Vitess docs: +// A hash vindex produces an 8-byte number. +// This means that all numbers less than 0x8000000000000000 will fall in shard -80. +// Any number with the highest bit set will be >= 0x8000000000000000, and will therefore +// belong to shard 80-. +// This means that from a keyrange perspective -80 == 00-80 == 0000-8000 == 000000-800000 +// If we don't do this padding, we could run into issues when transitioning from keyranges +// that use 2 bytes to 4 bytes. +func addPadding(bytes []byte) []byte { + paddedRange := bytes + + for i := len(bytes); i < 8; i++ { + paddedRange = append(paddedRange, 0) + } + return paddedRange } // KeyRangeStartSmaller returns true if right's keyrange start is _after_ left's start @@ -217,7 +235,7 @@ func KeyRangeStartEqual(left, right *topodatapb.KeyRange) bool { if right == nil { return len(left.Start) == 0 } - return bytes.Equal(left.Start, right.Start) + return bytes.Equal(addPadding(left.Start), addPadding(right.Start)) } // KeyRangeEndEqual returns true if both key ranges have the same end @@ -228,7 +246,7 @@ func KeyRangeEndEqual(left, right *topodatapb.KeyRange) bool { if right == nil { return len(left.End) == 0 } - return bytes.Equal(left.End, right.End) + return bytes.Equal(addPadding(left.End), addPadding(right.End)) } // For more info on the following functions, see: diff --git a/go/vt/key/key_test.go b/go/vt/key/key_test.go index f92d746f6e..d8d98e92f2 100644 --- a/go/vt/key/key_test.go +++ b/go/vt/key/key_test.go @@ -241,6 +241,167 @@ func TestKeyRangeAdd(t *testing.T) { } } +func TestKeyRangeEndEqual(t *testing.T) { + testcases := []struct { + first string + second string + out bool + }{{ + first: "", + second: "", + out: true, + }, { + first: "", + second: "-80", + out: false, + }, { + first: "40-", + second: "10-", + out: true, + }, { + first: "-8000", + second: "-80", + out: true, + }, { + first: "-8000", + second: "-8000000000000000", + out: true, + }, { + first: "-80", + second: "-8000", + out: true, + }} + stringToKeyRange := func(spec string) *topodatapb.KeyRange { + if spec == "" { + return nil + } + parts := strings.Split(spec, "-") + if len(parts) != 2 { + panic("invalid spec") + } + kr, err := ParseKeyRangeParts(parts[0], parts[1]) + if err != nil { + panic(err) + } + return kr + } + + for _, tcase := range testcases { + first := stringToKeyRange(tcase.first) + second := stringToKeyRange(tcase.second) + out := KeyRangeEndEqual(first, second) + if out != tcase.out { + t.Fatalf("KeyRangeEndEqual(%q, %q) expected %t, got %t", tcase.first, tcase.second, tcase.out, out) + } + } +} + +func TestKeyRangeStartEqual(t *testing.T) { + testcases := []struct { + first string + second string + out bool + }{{ + first: "", + second: "", + out: true, + }, { + first: "", + second: "-80", + out: true, + }, { + first: "40-", + second: "20-", + out: false, + }, { + first: "-8000", + second: "-80", + out: true, + }, { + first: "-8000", + second: "-8000000000000000", + out: true, + }, { + first: "-80", + second: "-8000", + out: true, + }} + stringToKeyRange := func(spec string) *topodatapb.KeyRange { + if spec == "" { + return nil + } + parts := strings.Split(spec, "-") + if len(parts) != 2 { + panic("invalid spec") + } + kr, err := ParseKeyRangeParts(parts[0], parts[1]) + if err != nil { + panic(err) + } + return kr + } + + for _, tcase := range testcases { + first := stringToKeyRange(tcase.first) + second := stringToKeyRange(tcase.second) + out := KeyRangeStartEqual(first, second) + if out != tcase.out { + t.Fatalf("KeyRangeStartEqual(%q, %q) expected %t, got %t", tcase.first, tcase.second, tcase.out, out) + } + } +} + +func TestKeyRangeEqual(t *testing.T) { + testcases := []struct { + first string + second string + out bool + }{{ + first: "", + second: "", + out: true, + }, { + first: "", + second: "-80", + out: false, + }, { + first: "-8000", + second: "-80", + out: true, + }, { + first: "-8000", + second: "-8000000000000000", + out: true, + }, { + first: "-80", + second: "-8000", + out: true, + }} + stringToKeyRange := func(spec string) *topodatapb.KeyRange { + if spec == "" { + return nil + } + parts := strings.Split(spec, "-") + if len(parts) != 2 { + panic("invalid spec") + } + kr, err := ParseKeyRangeParts(parts[0], parts[1]) + if err != nil { + panic(err) + } + return kr + } + + for _, tcase := range testcases { + first := stringToKeyRange(tcase.first) + second := stringToKeyRange(tcase.second) + out := KeyRangeEqual(first, second) + if out != tcase.out { + t.Fatalf("KeyRangeEqual(%q, %q) expected %t, got %t", tcase.first, tcase.second, tcase.out, out) + } + } +} + func TestEvenShardsKeyRange_Error(t *testing.T) { testCases := []struct { i, n int diff --git a/go/vt/topotools/split_test.go b/go/vt/topotools/split_test.go index 137eb4ce18..0ba13a3524 100644 --- a/go/vt/topotools/split_test.go +++ b/go/vt/topotools/split_test.go @@ -105,6 +105,18 @@ func TestValidateForReshard(t *testing.T) { sources: []string{"-80", "80-"}, targets: []string{"-40", "40-"}, out: "", + }, { + sources: []string{"52-53"}, + targets: []string{"5200-5240", "5240-5280", "5280-52c0", "52c0-5300"}, + out: "", + }, { + sources: []string{"5200-5300"}, + targets: []string{"520000-524000", "524000-528000", "528000-52c000", "52c000-530000"}, + out: "", + }, { + sources: []string{"-80", "80-"}, + targets: []string{"-4000000000000000", "4000000000000000-8000000000000000", "8000000000000000-80c0000000000000", "80c0000000000000-"}, + out: "", }, { sources: []string{"80-", "-80"}, targets: []string{"-40", "40-"}, From 1a4bf681a549b3e676cb7b09d61b6a8b8669b76a Mon Sep 17 00:00:00 2001 From: Rafael Chacon Date: Tue, 8 Jun 2021 15:56:12 -0700 Subject: [PATCH 44/64] Change name to be less confusing Signed-off-by: Rafael Chacon --- go/vt/key/key.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/go/vt/key/key.go b/go/vt/key/key.go index e271886012..57af2e570e 100644 --- a/go/vt/key/key.go +++ b/go/vt/key/key.go @@ -207,13 +207,13 @@ func KeyRangeEqual(left, right *topodatapb.KeyRange) bool { // This means that from a keyrange perspective -80 == 00-80 == 0000-8000 == 000000-800000 // If we don't do this padding, we could run into issues when transitioning from keyranges // that use 2 bytes to 4 bytes. -func addPadding(bytes []byte) []byte { - paddedRange := bytes +func addPadding(kr []byte) []byte { + paddedKr := kr - for i := len(bytes); i < 8; i++ { - paddedRange = append(paddedRange, 0) + for i := len(kr); i < 8; i++ { + paddedKr = append(paddedKr, 0) } - return paddedRange + return paddedKr } // KeyRangeStartSmaller returns true if right's keyrange start is _after_ left's start From 30e3739d008b0af3bcba6c783e44b3145753fdf1 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 8 Jun 2021 21:29:52 -0400 Subject: [PATCH 45/64] Add failing tests Signed-off-by: Andrew Mason --- go/vt/topo/topotests/cell_info_test.go | 49 ++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/go/vt/topo/topotests/cell_info_test.go b/go/vt/topo/topotests/cell_info_test.go index 7854763a68..eaae838efa 100644 --- a/go/vt/topo/topotests/cell_info_test.go +++ b/go/vt/topo/topotests/cell_info_test.go @@ -21,6 +21,7 @@ import ( "strings" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "context" @@ -176,4 +177,52 @@ func TestExpandCells(t *testing.T) { }) } + t.Run("aliases", func(t *testing.T) { + cells := []string{"cell1", "cell2", "cell3"} + ts := memorytopo.NewServer(cells...) + err := ts.CreateCellsAlias(ctx, "alias", &topodatapb.CellsAlias{Cells: cells}) + require.NoError(t, err) + + tests := []struct { + name string + in string + out []string + shouldErr bool + }{ + { + name: "alias only", + in: "alias", + out: []string{"cell1", "cell2", "cell3"}, + }, + { + name: "alias and cell in alias", // test deduping logic + in: "alias,cell1", + out: []string{"cell1", "cell2", "cell3"}, + }, + { + name: "just cells", + in: "cell1", + out: []string{"cell1"}, + }, + { + name: "missing alias", + in: "not_an_alias", + shouldErr: true, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + expanded, err := ts.ExpandCells(ctx, tt.in) + if tt.shouldErr { + assert.Error(t, err) + return + } + + require.NoError(t, err) + assert.ElementsMatch(t, expanded, tt.out) + }) + } + }) } From 5239e7f67fb9419d27eb8f2b3e87fdbf9a14d8ce Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 8 Jun 2021 21:31:27 -0400 Subject: [PATCH 46/64] [topo] Refactor `ExpandCells` to not error on valid aliases A couple other changes here: - Store the output cells in a set to dedupe the final slice. This covers the case where a caller passes `"alias,cell1"` where `cell1` is in the cell list for `alias`. Without a set, it would appear twice in the final result. - Add an inner function to correct `defer` semantics on the cancel funcs. Defering within the scope of a for loop is generally not what we want, since all of the deferred functions will pile up until the total function is done. Fixes #8290. Signed-off-by: Andrew Mason --- go/vt/topo/cell_info.go | 63 +++++++++++++++++++++++++---------------- 1 file changed, 39 insertions(+), 24 deletions(-) diff --git a/go/vt/topo/cell_info.go b/go/vt/topo/cell_info.go index 6ee6457586..9c1ec73120 100644 --- a/go/vt/topo/cell_info.go +++ b/go/vt/topo/cell_info.go @@ -17,12 +17,12 @@ limitations under the License. package topo import ( + "context" "path" "strings" "google.golang.org/protobuf/proto" - - "context" + "k8s.io/apimachinery/pkg/util/sets" "vitess.io/vitess/go/vt/vterrors" @@ -176,37 +176,52 @@ func (ts *Server) GetKnownCells(ctx context.Context) ([]string, error) { // ExpandCells takes a comma-separated list of cells and returns an array of cell names // Aliases are expanded and an empty string returns all cells func (ts *Server) ExpandCells(ctx context.Context, cells string) ([]string, error) { - var err error - var outputCells []string - inputCells := strings.Split(cells, ",") + var ( + err error + inputCells []string + outputCells = sets.NewString() // Use a set to dedupe if the input cells list includes an alias and a cell in that alias. + ) + if cells == "" { inputCells, err = ts.GetCellInfoNames(ctx) if err != nil { return nil, err } + } else { + inputCells = strings.Split(cells, ",") + } + + expandCell := func(ctx context.Context, cell string) error { + shortCtx, cancel := context.WithTimeout(ctx, *RemoteOperationTimeout) + defer cancel() + + _, err := ts.GetCellInfo(shortCtx, cell, false /* strongRead */) + if err != nil { + // Not a valid cell name. Check whether it is an alias. + shortCtx, cancel := context.WithTimeout(ctx, *RemoteOperationTimeout) + defer cancel() + + alias, err2 := ts.GetCellsAlias(shortCtx, cell, false /* strongRead */) + if err2 != nil { + return err // return the original err to indicate the cell does not exist + } + + // Expand the alias cells list into the final set. + outputCells.Insert(alias.Cells...) + return nil + } + + // Valid cell. + outputCells.Insert(cell) + return nil } for _, cell := range inputCells { cell2 := strings.TrimSpace(cell) - shortCtx, cancel := context.WithTimeout(ctx, *RemoteOperationTimeout) - defer cancel() - _, err := ts.GetCellInfo(shortCtx, cell2, false) - if err != nil { - // not a valid cell, check whether it is a cell alias - shortCtx, cancel := context.WithTimeout(ctx, *RemoteOperationTimeout) - defer cancel() - alias, err2 := ts.GetCellsAlias(shortCtx, cell2, false) - // if we get an error, either cellAlias doesn't exist or it isn't a cell alias at all. Ignore and continue - if err2 == nil { - outputCells = append(outputCells, alias.Cells...) - } - if err != nil { - return nil, err - } - } else { - // valid cell, add it to our list - outputCells = append(outputCells, cell2) + if err := expandCell(ctx, cell2); err != nil { + return nil, err } } - return outputCells, nil + + return outputCells.List(), nil } From 08eef0cb8163d2481ab0f1149d00d21b979b1a8b Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Wed, 9 Jun 2021 13:24:41 +0530 Subject: [PATCH 47/64] added function to check when the docker image is ready Signed-off-by: GuptaManan100 --- go/test/endtoend/docker/vttestserver.go | 43 +++++++++++++++++++- go/test/endtoend/docker/vttestserver_test.go | 13 +++--- 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/go/test/endtoend/docker/vttestserver.go b/go/test/endtoend/docker/vttestserver.go index 81f9266148..994c55b155 100644 --- a/go/test/endtoend/docker/vttestserver.go +++ b/go/test/endtoend/docker/vttestserver.go @@ -17,12 +17,14 @@ limitations under the License. package docker import ( + "encoding/json" "fmt" "os" "os/exec" "path" "strconv" "strings" + "time" ) const ( @@ -63,7 +65,7 @@ func (v *vttestserver) startDockerImage() error { cmd.Args = append(cmd.Args, "-e", fmt.Sprintf("NUM_SHARDS=%s", strings.Join(convertToStringSlice(v.numShards), ","))) cmd.Args = append(cmd.Args, "-e", "MYSQL_BIND_HOST=0.0.0.0") cmd.Args = append(cmd.Args, "-e", fmt.Sprintf("MYSQL_MAX_CONNECTIONS=%d", v.mysqlMaxConnecetions)) - cmd.Args = append(cmd.Args, `--health-cmd="mysqladmin ping -h127.0.0.1 -P33577"`) + cmd.Args = append(cmd.Args, "--health-cmd", "mysqladmin ping -h127.0.0.1 -P33577") cmd.Args = append(cmd.Args, "--health-interval=5s") cmd.Args = append(cmd.Args, "--health-timeout=2s") cmd.Args = append(cmd.Args, "--health-retries=5") @@ -76,6 +78,45 @@ func (v *vttestserver) startDockerImage() error { return nil } +// dockerStatus is a struct used to unmarshal json output from `docker inspect` +type dockerStatus struct { + State struct { + Health struct { + Status string + } + } +} + +// waitUntilDockerHealthy waits until the docker image is healthy. It takes in as argument the amount of seconds to wait before timeout +func (v *vttestserver) waitUntilDockerHealthy(timeoutDelay int) error { + timeOut := time.After(time.Duration(timeoutDelay) * time.Second) + + for { + select { + case <-timeOut: + // return error due to timeout + return fmt.Errorf("timed out waiting for docker image to start") + case <-time.After(time.Second): + cmd := exec.Command("docker", "inspect", "vttestserver-end2end-test") + out, err := cmd.Output() + if err != nil { + return err + } + var x []dockerStatus + err = json.Unmarshal(out, &x) + if err != nil { + return err + } + if len(x) > 0 { + status := x[0].State.Health.Status + if status == "healthy" { + return nil + } + } + } + } +} + // convertToStringSlice converts an integer slice to string slice func convertToStringSlice(intSlice []int) []string { var stringSlice []string diff --git a/go/test/endtoend/docker/vttestserver_test.go b/go/test/endtoend/docker/vttestserver_test.go index 4226f0c0c0..797e8f2e5c 100644 --- a/go/test/endtoend/docker/vttestserver_test.go +++ b/go/test/endtoend/docker/vttestserver_test.go @@ -21,7 +21,6 @@ import ( "fmt" "os" "testing" - "time" "github.com/google/go-cmp/cmp" @@ -53,7 +52,8 @@ func TestUnsharded(t *testing.T) { defer vtest.teardown() // wait for the docker to be setup - time.Sleep(10 * time.Second) + err = vtest.waitUntilDockerHealthy(10) + require.NoError(t, err) ctx := context.Background() vttestParams := mysql.ConnParams{ @@ -83,7 +83,8 @@ func TestSharded(t *testing.T) { defer vtest.teardown() // wait for the docker to be setup - time.Sleep(10 * time.Second) + err = vtest.waitUntilDockerHealthy(10) + require.NoError(t, err) ctx := context.Background() vttestParams := mysql.ConnParams{ @@ -115,7 +116,8 @@ func TestMysqlMaxCons(t *testing.T) { defer vtest.teardown() // wait for the docker to be setup - time.Sleep(10 * time.Second) + err = vtest.waitUntilDockerHealthy(10) + require.NoError(t, err) ctx := context.Background() vttestParams := mysql.ConnParams{ @@ -147,7 +149,8 @@ func TestLargeNumberOfKeyspaces(t *testing.T) { defer vtest.teardown() // wait for the docker to be setup - time.Sleep(15 * time.Second) + err = vtest.waitUntilDockerHealthy(15) + require.NoError(t, err) ctx := context.Background() vttestParams := mysql.ConnParams{ From 7257e15b434ecf157b87da09161ba705772284b9 Mon Sep 17 00:00:00 2001 From: Sara Bee <855595+doeg@users.noreply.github.com> Date: Wed, 9 Jun 2021 08:41:08 -0400 Subject: [PATCH 48/64] [vtadmin-web] Add client-side error handling interface + Bugsnag implementation Signed-off-by: Sara Bee <855595+doeg@users.noreply.github.com> --- web/vtadmin/package-lock.json | 70 +++++++++++ web/vtadmin/package.json | 1 + web/vtadmin/src/api/http.test.ts | 62 ++++++--- web/vtadmin/src/api/http.ts | 115 ++++++++--------- web/vtadmin/src/api/responseTypes.ts | 28 +++++ web/vtadmin/src/errors/bugsnag.ts | 49 ++++++++ web/vtadmin/src/errors/errorHandler.test.ts | 126 +++++++++++++++++++ web/vtadmin/src/errors/errorHandler.ts | 72 +++++++++++ web/vtadmin/src/errors/errorHandlers.ts | 24 ++++ web/vtadmin/src/errors/errorTypes.ts | 131 ++++++++++++++++++++ web/vtadmin/src/index.tsx | 3 + web/vtadmin/src/react-app-env.d.ts | 15 ++- 12 files changed, 624 insertions(+), 72 deletions(-) create mode 100644 web/vtadmin/src/api/responseTypes.ts create mode 100644 web/vtadmin/src/errors/bugsnag.ts create mode 100644 web/vtadmin/src/errors/errorHandler.test.ts create mode 100644 web/vtadmin/src/errors/errorHandler.ts create mode 100644 web/vtadmin/src/errors/errorHandlers.ts create mode 100644 web/vtadmin/src/errors/errorTypes.ts diff --git a/web/vtadmin/package-lock.json b/web/vtadmin/package-lock.json index 61dd23b7fe..f9a1043d37 100644 --- a/web/vtadmin/package-lock.json +++ b/web/vtadmin/package-lock.json @@ -1137,6 +1137,58 @@ "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz", "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==" }, + "@bugsnag/browser": { + "version": "7.10.1", + "resolved": "https://registry.npmjs.org/@bugsnag/browser/-/browser-7.10.1.tgz", + "integrity": "sha512-Yxm/DheT/NHX2PhadBDuafuHBhP547Iav6Y9jf+skBBSi1n0ZYkGhtVxh8ZWLgqz5W8MsJ0HFiLBqcg/mulSvQ==", + "requires": { + "@bugsnag/core": "^7.10.0" + } + }, + "@bugsnag/core": { + "version": "7.10.0", + "resolved": "https://registry.npmjs.org/@bugsnag/core/-/core-7.10.0.tgz", + "integrity": "sha512-sDa2nDxwsxHQx2/2/tsBWjYqH0TewCR8N/r5at6B+irwVkI0uts7Qc2JyqDTfiEiBXKVEXFK+fHTz1x9b8tsiA==", + "requires": { + "@bugsnag/cuid": "^3.0.0", + "@bugsnag/safe-json-stringify": "^6.0.0", + "error-stack-parser": "^2.0.3", + "iserror": "0.0.2", + "stack-generator": "^2.0.3" + } + }, + "@bugsnag/cuid": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@bugsnag/cuid/-/cuid-3.0.0.tgz", + "integrity": "sha512-LOt8aaBI+KvOQGneBtpuCz3YqzyEAehd1f3nC5yr9TIYW1+IzYKa2xWS4EiMz5pPOnRPHkyyS5t/wmSmN51Gjg==" + }, + "@bugsnag/js": { + "version": "7.10.1", + "resolved": "https://registry.npmjs.org/@bugsnag/js/-/js-7.10.1.tgz", + "integrity": "sha512-1/MK/Bw2ViFx1hMG2TOX8MOq/LzT2VRd0VswknF4LYsZSgzohkRzz/hi6P2TSlLeapRs+bkDC6u2RCq4zYvyiA==", + "requires": { + "@bugsnag/browser": "^7.10.1", + "@bugsnag/node": "^7.10.1" + } + }, + "@bugsnag/node": { + "version": "7.10.1", + "resolved": "https://registry.npmjs.org/@bugsnag/node/-/node-7.10.1.tgz", + "integrity": "sha512-kpasrz/im5ljptt2JOqrjbOu4b0i5sAZOYU4L0psWXlD31/wXytk7im11QlNALdI8gZZBxIFsVo8ks6dR6mHzg==", + "requires": { + "@bugsnag/core": "^7.10.0", + "byline": "^5.0.0", + "error-stack-parser": "^2.0.2", + "iserror": "^0.0.2", + "pump": "^3.0.0", + "stack-generator": "^2.0.3" + } + }, + "@bugsnag/safe-json-stringify": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/@bugsnag/safe-json-stringify/-/safe-json-stringify-6.0.0.tgz", + "integrity": "sha512-htzFO1Zc57S8kgdRK9mLcPVTW1BY2ijfH7Dk2CeZmspTWKdKqSo1iwmqrq2WtRjFlo8aRZYgLX0wFrDXF/9DLA==" + }, "@cnakazawa/watch": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/@cnakazawa/watch/-/watch-1.0.4.tgz", @@ -4047,6 +4099,11 @@ "resolved": "https://registry.npmjs.org/builtin-status-codes/-/builtin-status-codes-3.0.0.tgz", "integrity": "sha1-hZgoeOIbmOHGZCXgPQF0eI9Wnug=" }, + "byline": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/byline/-/byline-5.0.0.tgz", + "integrity": "sha1-dBxSFkaOrcRXsDQQEYrXfejB3bE=" + }, "bytes": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", @@ -8517,6 +8574,11 @@ "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" }, + "iserror": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/iserror/-/iserror-0.0.2.tgz", + "integrity": "sha1-vVNFH+L2aLnyQCwZZnh6qix8C/U=" + }, "isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", @@ -15754,6 +15816,14 @@ "resolved": "https://registry.npmjs.org/stable/-/stable-0.1.8.tgz", "integrity": "sha512-ji9qxRnOVfcuLDySj9qzhGSEFVobyt1kIOSkj1qZzYLzq7Tos/oUUWvotUPQLlrsidqsK6tBH89Bc9kL5zHA6w==" }, + "stack-generator": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/stack-generator/-/stack-generator-2.0.5.tgz", + "integrity": "sha512-/t1ebrbHkrLrDuNMdeAcsvynWgoH/i4o8EGGfX7dEYDoTXOYVAkEpFdtshlvabzc6JlJ8Kf9YdFEoz7JkzGN9Q==", + "requires": { + "stackframe": "^1.1.1" + } + }, "stack-utils": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-2.0.3.tgz", diff --git a/web/vtadmin/package.json b/web/vtadmin/package.json index 3679c2f3fb..255f5ca8a9 100644 --- a/web/vtadmin/package.json +++ b/web/vtadmin/package.json @@ -7,6 +7,7 @@ "npm": ">=6.14.9" }, "dependencies": { + "@bugsnag/js": "^7.10.1", "@testing-library/user-event": "^12.6.0", "@types/classnames": "^2.2.11", "@types/jest": "^26.0.19", diff --git a/web/vtadmin/src/api/http.test.ts b/web/vtadmin/src/api/http.test.ts index af46c15b26..b5ca9fba40 100644 --- a/web/vtadmin/src/api/http.test.ts +++ b/web/vtadmin/src/api/http.test.ts @@ -17,8 +17,10 @@ import { rest } from 'msw'; import { setupServer } from 'msw/node'; import * as api from './http'; -import { vtadmin as pb } from '../proto/vtadmin'; -import { HTTP_RESPONSE_NOT_OK_ERROR, MALFORMED_HTTP_RESPONSE_ERROR } from './http'; +import { HTTP_RESPONSE_NOT_OK_ERROR, MALFORMED_HTTP_RESPONSE_ERROR } from '../errors/errorTypes'; +import * as errorHandler from '../errors/errorHandler'; + +jest.mock('../errors/errorHandler'); // This test suite uses Mock Service Workers (https://github.com/mswjs/msw) // to mock HTTP responses from vtadmin-api. @@ -56,7 +58,9 @@ const TEST_PROCESS_ENV = { }; beforeAll(() => { - process.env = { ...TEST_PROCESS_ENV }; + // TypeScript can get a little cranky with the automatic + // string/boolean type conversions, hence this cast. + process.env = { ...TEST_PROCESS_ENV } as NodeJS.ProcessEnv; // Enable API mocking before tests. server.listen(); @@ -64,7 +68,7 @@ beforeAll(() => { afterEach(() => { // Reset the process.env to clear out any changes made in the tests. - process.env = { ...TEST_PROCESS_ENV }; + process.env = { ...TEST_PROCESS_ENV } as NodeJS.ProcessEnv; jest.restoreAllMocks(); @@ -92,34 +96,52 @@ describe('api/http', () => { it('throws an error if response.ok is false', async () => { const endpoint = `/api/tablets`; - const response = { ok: false }; - mockServerJson(endpoint, response); + const response = { + ok: false, + error: { + code: 'oh_no', + message: 'something went wrong', + }, + }; - expect.assertions(3); + // See https://mswjs.io/docs/recipes/mocking-error-responses + server.use(rest.get(endpoint, (req, res, ctx) => res(ctx.status(500), ctx.json(response)))); + + expect.assertions(5); try { await api.fetchTablets(); } catch (e) { /* eslint-disable jest/no-conditional-expect */ expect(e.name).toEqual(HTTP_RESPONSE_NOT_OK_ERROR); - expect(e.message).toEqual(endpoint); + expect(e.message).toEqual('[status 500] /api/tablets: oh_no something went wrong'); expect(e.response).toEqual(response); + + expect(errorHandler.notify).toHaveBeenCalledTimes(1); + expect(errorHandler.notify).toHaveBeenCalledWith(e); /* eslint-enable jest/no-conditional-expect */ } }); it('throws an error on malformed JSON', async () => { const endpoint = `/api/tablets`; - server.use(rest.get(endpoint, (req, res, ctx) => res(ctx.body('this is fine')))); + server.use( + rest.get(endpoint, (req, res, ctx) => + res(ctx.status(504), ctx.body('504 Gateway Time-out')) + ) + ); - expect.assertions(2); + expect.assertions(4); try { await api.vtfetch(endpoint); } catch (e) { /* eslint-disable jest/no-conditional-expect */ - expect(e.name).toEqual('SyntaxError'); - expect(e.message.startsWith('Unexpected token')).toBeTruthy(); + expect(e.name).toEqual(MALFORMED_HTTP_RESPONSE_ERROR); + expect(e.message).toEqual('[status 504] /api/tablets: Unexpected token < in JSON at position 0'); + + expect(errorHandler.notify).toHaveBeenCalledTimes(1); + expect(errorHandler.notify).toHaveBeenCalledWith(e); /* eslint-enable jest/no-conditional-expect */ } }); @@ -151,7 +173,9 @@ describe('api/http', () => { await api.vtfetch(endpoint); expect(global.fetch).toHaveBeenCalledTimes(1); - expect(global.fetch).toHaveBeenCalledWith(endpoint, { credentials: 'include' }); + expect(global.fetch).toHaveBeenCalledWith(endpoint, { + credentials: 'include', + }); jest.restoreAllMocks(); }); @@ -165,7 +189,9 @@ describe('api/http', () => { await api.vtfetch(endpoint); expect(global.fetch).toHaveBeenCalledTimes(1); - expect(global.fetch).toHaveBeenCalledWith(endpoint, { credentials: undefined }); + expect(global.fetch).toHaveBeenCalledWith(endpoint, { + credentials: undefined, + }); jest.restoreAllMocks(); }); @@ -187,6 +213,9 @@ describe('api/http', () => { 'Invalid fetch credentials property: nope. Must be undefined or one of omit, same-origin, include' ); expect(global.fetch).toHaveBeenCalledTimes(0); + + expect(errorHandler.notify).toHaveBeenCalledTimes(1); + expect(errorHandler.notify).toHaveBeenCalledWith(e); /* eslint-enable jest/no-conditional-expect */ } @@ -200,7 +229,7 @@ describe('api/http', () => { const endpoint = '/api/foos'; mockServerJson(endpoint, { ok: true, result: { foos: null } }); - expect.assertions(1); + expect.assertions(3); try { await api.vtfetchEntities({ @@ -211,6 +240,9 @@ describe('api/http', () => { } catch (e) { /* eslint-disable jest/no-conditional-expect */ expect(e.message).toMatch('expected entities to be an array, got null'); + + expect(errorHandler.notify).toHaveBeenCalledTimes(1); + expect(errorHandler.notify).toHaveBeenCalledWith(e); /* eslint-enable jest/no-conditional-expect */ } }); diff --git a/web/vtadmin/src/api/http.ts b/web/vtadmin/src/api/http.ts index 8c2f0c7198..61c8c62c0e 100644 --- a/web/vtadmin/src/api/http.ts +++ b/web/vtadmin/src/api/http.ts @@ -15,68 +15,66 @@ */ import { vtadmin as pb } from '../proto/vtadmin'; +import * as errorHandler from '../errors/errorHandler'; +import { HttpFetchError, HttpResponseNotOkError, MalformedHttpResponseError } from '../errors/errorTypes'; +import { HttpOkResponse } from './responseTypes'; -interface HttpOkResponse { - ok: true; - result: any; -} - -interface HttpErrorResponse { - ok: false; -} - -export const MALFORMED_HTTP_RESPONSE_ERROR = 'MalformedHttpResponseError'; - -// MalformedHttpResponseError is thrown when the JSON response envelope -// is an unexpected shape. -class MalformedHttpResponseError extends Error { - responseJson: object; - - constructor(message: string, responseJson: object) { - super(message); - this.name = MALFORMED_HTTP_RESPONSE_ERROR; - this.responseJson = responseJson; - } -} - -export const HTTP_RESPONSE_NOT_OK_ERROR = 'HttpResponseNotOkError'; - -// HttpResponseNotOkError is throw when the `ok` is false in -// the JSON response envelope. -class HttpResponseNotOkError extends Error { - response: HttpErrorResponse | null; - - constructor(endpoint: string, response: HttpErrorResponse) { - super(endpoint); - this.name = HTTP_RESPONSE_NOT_OK_ERROR; - this.response = response; - } -} - -// vtfetch makes HTTP requests against the given vtadmin-api endpoint -// and returns the parsed response. -// -// HttpResponse envelope types are not defined in vtadmin.proto (nor should they be) -// thus we have to validate the shape of the API response with more care. -// -// Note that this only validates the HttpResponse envelope; it does not -// do any type checking or validation on the result. +/** + * vtfetch makes HTTP requests against the given vtadmin-api endpoint + * and returns the parsed response. + * + * HttpResponse envelope types are not defined in vtadmin.proto (nor should they be) + * thus we have to validate the shape of the API response with more care. + * + * Note that this only validates the HttpResponse envelope; it does not + * do any type checking or validation on the result. + */ export const vtfetch = async (endpoint: string): Promise => { - const { REACT_APP_VTADMIN_API_ADDRESS } = process.env; + try { + const { REACT_APP_VTADMIN_API_ADDRESS } = process.env; - const url = `${REACT_APP_VTADMIN_API_ADDRESS}${endpoint}`; - const opts = vtfetchOpts(); + const url = `${REACT_APP_VTADMIN_API_ADDRESS}${endpoint}`; + const opts = vtfetchOpts(); - const response = await global.fetch(url, opts); + let response = null; + try { + response = await global.fetch(url, opts); + } catch (error) { + // Capture fetch() promise rejections and rethrow as HttpFetchError. + // fetch() promises will reject with a TypeError when a network error is + // encountered or CORS is misconfigured, in which case the request never + // makes it to the server. + // See https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API/Using_Fetch#checking_that_the_fetch_was_successful + throw new HttpFetchError(url); + } - const json = await response.json(); - if (!('ok' in json)) throw new MalformedHttpResponseError('invalid http envelope', json); + let json = null; + try { + json = await response.json(); + } catch (error) { + throw new MalformedHttpResponseError(error.message, endpoint, json, response); + } - // Throw "not ok" responses so that react-query correctly interprets them as errors. - // See https://react-query.tanstack.com/guides/query-functions#handling-and-throwing-errors - if (!json.ok) throw new HttpResponseNotOkError(endpoint, json); + if (!('ok' in json)) { + throw new MalformedHttpResponseError('invalid HTTP envelope', endpoint, json, response); + } - return json as HttpOkResponse; + if (!json.ok) { + throw new HttpResponseNotOkError(endpoint, json, response); + } + + return json as HttpOkResponse; + } catch (error) { + // Most commonly, react-query is the downstream consumer of + // errors thrown in vtfetch. Because react-query "handles" errors + // by propagating them to components (as it should!), any errors thrown + // from vtfetch are _not_ automatically logged as "unhandled errors". + // Instead, we catch errors and manually notify our error handling serivce(s), + // and then rethrow the error for react-query to propagate the usual way. + // See https://react-query.tanstack.com/guides/query-functions#handling-and-throwing-errors + errorHandler.notify(error); + throw error; + } }; export const vtfetchOpts = (): RequestInit => { @@ -106,7 +104,12 @@ export const vtfetchEntities = async (opts: { const entities = opts.extract(res); if (!Array.isArray(entities)) { - throw Error(`expected entities to be an array, got ${entities}`); + // Since react-query is the downstream consumer of vtfetch + vtfetchEntities, + // errors thrown in either function will be "handled" and will not automatically + // propagate as "unhandled" errors, meaning we have to log them manually. + const error = Error(`expected entities to be an array, got ${entities}`); + errorHandler.notify(error); + throw error; } return entities.map(opts.transform); diff --git a/web/vtadmin/src/api/responseTypes.ts b/web/vtadmin/src/api/responseTypes.ts new file mode 100644 index 0000000000..32be2d2bb5 --- /dev/null +++ b/web/vtadmin/src/api/responseTypes.ts @@ -0,0 +1,28 @@ +/** + * Copyright 2021 The Vitess Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export interface HttpOkResponse { + ok: true; + result: any; +} + +export interface HttpErrorResponse { + error?: { + message?: string; + code?: string; + }; + ok: false; +} diff --git a/web/vtadmin/src/errors/bugsnag.ts b/web/vtadmin/src/errors/bugsnag.ts new file mode 100644 index 0000000000..4a985dee3f --- /dev/null +++ b/web/vtadmin/src/errors/bugsnag.ts @@ -0,0 +1,49 @@ +/** + * Copyright 2021 The Vitess Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import BugsnagJS from '@bugsnag/js'; +import { ErrorHandler } from './errorTypes'; + +const { REACT_APP_BUGSNAG_API_KEY } = process.env; + +/** + * If using Bugsnag, Bugsnag.start() will automatically capture and report + * unhandled exceptions and unhandled promise rejections, as well as + * initialize it for capturing handled errors. + */ +export const initialize = () => { + if (typeof REACT_APP_BUGSNAG_API_KEY === 'string') { + BugsnagJS.start(REACT_APP_BUGSNAG_API_KEY); + } +}; + +export const isEnabled = () => typeof REACT_APP_BUGSNAG_API_KEY === 'string'; + +export const notify = (error: Error, env: object, metadata?: object) => { + // See https://docs.bugsnag.com/platforms/javascript/reporting-handled-errors/ + BugsnagJS.notify(error, (event) => { + event.addMetadata('env', env); + + if (!!metadata) { + event.addMetadata('metadata', metadata); + } + }); +}; + +export const Bugsnag: ErrorHandler = { + initialize, + isEnabled, + notify, +}; diff --git a/web/vtadmin/src/errors/errorHandler.test.ts b/web/vtadmin/src/errors/errorHandler.test.ts new file mode 100644 index 0000000000..929daae20f --- /dev/null +++ b/web/vtadmin/src/errors/errorHandler.test.ts @@ -0,0 +1,126 @@ +/** + * Copyright 2021 The Vitess Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ErrorHandler, HttpResponseNotOkError } from './errorTypes'; +import * as errorHandler from './errorHandler'; +import * as errorHandlers from './errorHandlers'; + +// Since vtadmin uses process.env variables quite a bit, we need to +// do a bit of a dance to clear them out between test runs. +const ORIGINAL_PROCESS_ENV = process.env; +const TEST_PROCESS_ENV = { + ...process.env, + REACT_APP_VTADMIN_API_ADDRESS: '', +}; + +beforeAll(() => { + // TypeScript can get a little cranky with the automatic + // string/boolean type conversions, hence this cast. + process.env = { ...TEST_PROCESS_ENV } as NodeJS.ProcessEnv; +}); + +afterEach(() => { + // Reset the process.env to clear out any changes made in the tests. + process.env = { ...TEST_PROCESS_ENV } as NodeJS.ProcessEnv; + + jest.restoreAllMocks(); +}); + +afterAll(() => { + process.env = { ...ORIGINAL_PROCESS_ENV }; +}); + +describe('errorHandler', () => { + let mockErrorHandler: ErrorHandler; + let mockEnv: NodeJS.ProcessEnv; + + beforeEach(() => { + mockErrorHandler = { + initialize: jest.fn(), + isEnabled: () => true, + notify: jest.fn(), + }; + + jest.spyOn(errorHandlers, 'getHandlers').mockReturnValue([mockErrorHandler]); + + mockEnv = { + REACT_APP_VTADMIN_API_ADDRESS: 'http://example.com', + } as NodeJS.ProcessEnv; + process.env = mockEnv; + }); + + describe('initialize', () => { + it('initializes enabled handlers', () => { + errorHandler.initialize(); + expect(mockErrorHandler.initialize).toHaveBeenCalledTimes(1); + }); + }); + + describe('notify', () => { + it('notifies enabled ErrorHandlers', () => { + const err = new Error('testing'); + errorHandler.notify(err); + + expect(mockErrorHandler.notify).toHaveBeenCalledTimes(1); + expect(mockErrorHandler.notify).toHaveBeenCalledWith(err, mockEnv, { + errorMetadata: {}, + }); + }); + + it("appends metadata from the Error's instance properties", () => { + const response = new Response('', { status: 500 }); + const err = new HttpResponseNotOkError('/api/test', { ok: false }, response); + errorHandler.notify(err, { goodbye: 'moon' }); + + expect(mockErrorHandler.notify).toHaveBeenCalledTimes(1); + expect(mockErrorHandler.notify).toHaveBeenCalledWith(err, mockEnv, { + errorMetadata: { + fetchResponse: { + ok: false, + status: 500, + statusText: '', + type: 'default', + url: '', + }, + name: 'HttpResponseNotOkError', + response: { ok: false }, + }, + goodbye: 'moon', + }); + }); + + it('only includes santizied environment variables', () => { + process.env = { + REACT_APP_VTADMIN_API_ADDRESS: 'http://not-secret.example.com', + REACT_APP_BUGSNAG_API_KEY: 'secret', + } as NodeJS.ProcessEnv; + + const err = new Error('testing'); + errorHandler.notify(err); + + expect(mockErrorHandler.notify).toHaveBeenCalledTimes(1); + expect(mockErrorHandler.notify).toHaveBeenCalledWith( + err, + { + REACT_APP_VTADMIN_API_ADDRESS: 'http://not-secret.example.com', + }, + { + errorMetadata: {}, + } + ); + }); + }); +}); diff --git a/web/vtadmin/src/errors/errorHandler.ts b/web/vtadmin/src/errors/errorHandler.ts new file mode 100644 index 0000000000..abc4faa71d --- /dev/null +++ b/web/vtadmin/src/errors/errorHandler.ts @@ -0,0 +1,72 @@ +/** + * Copyright 2021 The Vitess Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { pick } from 'lodash'; +import { getHandlers } from './errorHandlers'; + +/** + * Initializes error handling for both unhandled and handled exceptions. + * This should be called as early as possible. + */ +export const initialize = () => { + getHandlers().forEach((h) => h.initialize()); +}; + +/** + * Manually notify error handlers of an error. Also known as + * a "handled error". + * + * @param error - The Error that was thrown/captured. + * @param metadata - Additional key/value metadata to log. Note that + * additional metadata from `error` will be added to the metadata + * object under the key "errorMetadata" before it is passed along + * to the active ErrorHandler clients(s). + */ +export const notify = (error: Error, metadata?: object) => { + const env = sanitizeEnv(); + const errorMetadata = Object.getOwnPropertyNames(error).reduce((acc, propertyName) => { + // Only annotate the `metadata` object with properties beyond the standard instance + // properties. (Bugsnag, for example, does not log additional `Error` properties: + // they have to be logged as additional metadata.) + if (propertyName !== 'stack' && propertyName !== 'message') { + acc[propertyName] = (error as any)[propertyName]; + } + + return acc; + }, {} as { [k: string]: any }); + + getHandlers().forEach((h) => + h.notify(error, env, { + errorMetadata, + ...metadata, + }) + ); +}; + +/** + * sanitizeEnv serializes process.env into an object that's sent to + * configured error handlers, for extra debugging context. + * Implemented as an allow list, rather than as a block list, to avoid + * leaking sensitive environment variables, like API keys. + */ +const sanitizeEnv = () => + pick(process.env, [ + 'REACT_APP_BUILD_BRANCH', + 'REACT_APP_BUILD_SHA', + 'REACT_APP_ENABLE_EXPERIMENTAL_TABLET_DEBUG_VARS', + 'REACT_APP_FETCH_CREDENTIALS', + 'REACT_APP_VTADMIN_API_ADDRESS', + ]); diff --git a/web/vtadmin/src/errors/errorHandlers.ts b/web/vtadmin/src/errors/errorHandlers.ts new file mode 100644 index 0000000000..9bf74259a9 --- /dev/null +++ b/web/vtadmin/src/errors/errorHandlers.ts @@ -0,0 +1,24 @@ +/** + * Copyright 2021 The Vitess Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as Bugsnag from './bugsnag'; +import { ErrorHandler } from './errorTypes'; + +/** + * getHandlers returns a list of enabled error handlers. + * This is implemented in its own file to make testing easier. + */ +export const getHandlers = (): ErrorHandler[] => [Bugsnag].filter((h) => h.isEnabled()); diff --git a/web/vtadmin/src/errors/errorTypes.ts b/web/vtadmin/src/errors/errorTypes.ts new file mode 100644 index 0000000000..dd207cde23 --- /dev/null +++ b/web/vtadmin/src/errors/errorTypes.ts @@ -0,0 +1,131 @@ +/** + * Copyright 2021 The Vitess Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { HttpErrorResponse } from '../api/responseTypes'; + +/** + * ErrorHandler defines a common interface for bindings to + * client-side monitoring services. + * + * Similar to Vitess itself [1], vtadmin-web can integrate with a variety + * of client-side monitoring tools. At the time of this writing, only + * Bugsnag integration is added. This interface, however, allows flexibility + * to implement other monitoring serivces, like Sentry, Datadog, etc. + * + * [1] https://vitess.io/docs/user-guides/configuration-basic/monitoring/ + */ +export interface ErrorHandler { + /** + * Handler to initialize the monitoring client. Called at the very + * beginning of the app life cycle. If a particular client supports + * capturing unhandled exceptions (as most do) that initialization + * logic should happen here. + */ + initialize: () => void; + + /** + * Handler to determine whether the monitoring client is enabled. + */ + isEnabled: () => boolean; + + /** + * Handler to manually notify the monitoring system of a problem. + * + * @param error - The Error that was thrown + * @param env - Sanitized process.env environment variables + * @param metadata - Additional, arbitrary metadata. + */ + notify: (error: Error, env: object, metadata?: object) => void; +} + +interface SerializedFetchResponse { + ok: boolean; + status: number; + statusText: string; + type: string; + url: string; +} + +/** + * serializeFetchResponse serializes a Response object into + * a simplified JSON object. This is particularly useful when + * logging and/or sending errors to monitoring clients, as + * a full Response object JSON.stringifies to "{}"." + */ +export const serializeFetchResponse = (fetchResponse: Response) => ({ + ok: fetchResponse.ok, + status: fetchResponse.status, + statusText: fetchResponse.statusText, + type: fetchResponse.type, + url: fetchResponse.url, +}); + +export const MALFORMED_HTTP_RESPONSE_ERROR = 'MalformedHttpResponseError'; + +/** + * MalformedHttpResponseError is thrown when the JSON response envelope + * is an unexpected shape. + */ +export class MalformedHttpResponseError extends Error { + fetchResponse: SerializedFetchResponse; + responseJson: object; + + constructor(message: string, endpoint: string, responseJson: object, fetchResponse: Response) { + const key = `[status ${fetchResponse.status}] ${endpoint}: ${message}`; + super(key); + + this.name = MALFORMED_HTTP_RESPONSE_ERROR; + this.responseJson = responseJson; + this.fetchResponse = serializeFetchResponse(fetchResponse); + } +} + +export const HTTP_RESPONSE_NOT_OK_ERROR = 'HttpResponseNotOkError'; + +/** + * HttpResponseNotOkError is throw when the `ok` is false in + * the JSON response envelope. + * + * See https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API/Using_Fetch#checking_that_the_fetch_was_successful + */ +export class HttpResponseNotOkError extends Error { + fetchResponse: SerializedFetchResponse; + response: HttpErrorResponse | null; + + constructor(endpoint: string, response: HttpErrorResponse, fetchResponse: Response) { + const key = `[status ${fetchResponse.status}] ${endpoint}: ${response.error?.code} ${response.error?.message}`; + super(key); + + this.name = HTTP_RESPONSE_NOT_OK_ERROR; + this.response = response; + this.fetchResponse = serializeFetchResponse(fetchResponse); + } +} + +export const HTTP_FETCH_ERROR = 'HttpFetchError'; + +/** + * HttpFetchError is thrown when fetch() promises reject with a TypeError when a network error is + * encountered or CORS is misconfigured. + * + * See https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API/Using_Fetch#checking_that_the_fetch_was_successful + */ +export class HttpFetchError extends Error { + constructor(endpoint: string) { + super(endpoint); + this.name = HTTP_FETCH_ERROR; + } +} diff --git a/web/vtadmin/src/index.tsx b/web/vtadmin/src/index.tsx index 9b7d9eae8d..352ee9faa6 100644 --- a/web/vtadmin/src/index.tsx +++ b/web/vtadmin/src/index.tsx @@ -21,6 +21,9 @@ import './index.css'; import './components/charts/charts.scss'; import { App } from './components/App'; +import * as errorHandler from './errors/errorHandler'; + +errorHandler.initialize(); const queryClient = new QueryClient(); diff --git a/web/vtadmin/src/react-app-env.d.ts b/web/vtadmin/src/react-app-env.d.ts index 4b74c3ea54..179e1d3e0e 100644 --- a/web/vtadmin/src/react-app-env.d.ts +++ b/web/vtadmin/src/react-app-env.d.ts @@ -4,13 +4,26 @@ declare namespace NodeJS { NODE_ENV: 'development' | 'production' | 'test'; PUBLIC_URL: string; + /* REQUIRED */ + // Required. The full address of vtadmin-api's HTTP interface. // Example: "http://127.0.0.1:12345" REACT_APP_VTADMIN_API_ADDRESS: string; + /* OPTIONAL */ + + // Optional. An API key for https://bugsnag.com. If defined, + // the @bugsnag/js client will be initialized. Your Bugsnag API key + // can be found in your Bugsnag Project Settings. + REACT_APP_BUGSNAG_API_KEY?: string; + + // Optional. Build variables. + REACT_APP_BUILD_BRANCH?: string; + REACT_APP_BUILD_SHA?: string; + // Optional, but recommended. When true, enables front-end components that query // vtadmin-api's /api/experimental/tablet/{tablet}/debug/vars endpoint. - REACT_APP_ENABLE_EXPERIMENTAL_TABLET_DEBUG_VARS: boolean; + REACT_APP_ENABLE_EXPERIMENTAL_TABLET_DEBUG_VARS?: boolean | string; // Optional. Configures the `credentials` property for fetch requests. // made against vtadmin-api. If unspecified, uses fetch defaults. From 248909bfdd6a204fec4a0d2ec4158b390ded5aad Mon Sep 17 00:00:00 2001 From: Rafael Chacon Date: Wed, 9 Jun 2021 10:59:08 -0700 Subject: [PATCH 49/64] Avoid races Signed-off-by: Rafael Chacon --- go/vt/key/key.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/go/vt/key/key.go b/go/vt/key/key.go index 57af2e570e..523d154042 100644 --- a/go/vt/key/key.go +++ b/go/vt/key/key.go @@ -205,10 +205,14 @@ func KeyRangeEqual(left, right *topodatapb.KeyRange) bool { // Any number with the highest bit set will be >= 0x8000000000000000, and will therefore // belong to shard 80-. // This means that from a keyrange perspective -80 == 00-80 == 0000-8000 == 000000-800000 -// If we don't do this padding, we could run into issues when transitioning from keyranges +// If we don't add this padding, we could run into issues when transitioning from keyranges // that use 2 bytes to 4 bytes. func addPadding(kr []byte) []byte { - paddedKr := kr + paddedKr := make([]byte, 8) + + for i := 0; i < len(kr); i++ { + paddedKr = append(paddedKr, kr[i]) + } for i := len(kr); i < 8; i++ { paddedKr = append(paddedKr, 0) From c3fb79c44f6d45413075c1629ae94a7272a7f857 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Wed, 9 Jun 2021 15:12:25 -0400 Subject: [PATCH 50/64] Correct copyright year and comment typo Signed-off-by: Andrew Mason --- go/cmd/vtctldclient/plugin_grpcvtctlclient.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/cmd/vtctldclient/plugin_grpcvtctlclient.go b/go/cmd/vtctldclient/plugin_grpcvtctlclient.go index e08d657f42..48c631a8ba 100644 --- a/go/cmd/vtctldclient/plugin_grpcvtctlclient.go +++ b/go/cmd/vtctldclient/plugin_grpcvtctlclient.go @@ -1,5 +1,5 @@ /* -Copyright 2020 The Vitess Authors. +Copyright 2021 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ limitations under the License. package main -// Imports and register the gRPC vtctl client. +// Imports and registers the gRPC vtctl client. import ( _ "vitess.io/vitess/go/vt/vtctl/grpcvtctlclient" From f964e01ec666a57b0f648933c9b3eec01c281dfb Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Wed, 9 Jun 2021 15:51:02 -0400 Subject: [PATCH 51/64] Use shared testing helper which guards against races in tmclient construction Fixes #8299 Signed-off-by: Andrew Mason --- go/vt/vtctl/grpcvtctldserver/server_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/go/vt/vtctl/grpcvtctldserver/server_test.go b/go/vt/vtctl/grpcvtctldserver/server_test.go index 21a2fa2180..3fb15bf0f1 100644 --- a/go/vt/vtctl/grpcvtctldserver/server_test.go +++ b/go/vt/vtctl/grpcvtctldserver/server_test.go @@ -304,7 +304,9 @@ func TestApplyRoutingRules(t *testing.T) { factory.SetError(errors.New("topo down for testing")) } - vtctld := NewVtctldServer(ts) + vtctld := testutil.NewVtctldServerWithTabletManagerClient(t, ts, nil, func(ts *topo.Server) vtctlservicepb.VtctldServer { + return NewVtctldServer(ts) + }) _, err := vtctld.ApplyRoutingRules(ctx, tt.req) if tt.shouldErr { assert.Error(t, err) From cfed06207fce627e331bf45b4e6c9c3d9479c181 Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Thu, 10 Jun 2021 07:56:41 +0200 Subject: [PATCH 52/64] Fixing TestVSchemaTrackerInitAndUpdate E2E test Signed-off-by: Florent Poinsard --- go/test/endtoend/vtgate/misc_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/go/test/endtoend/vtgate/misc_test.go b/go/test/endtoend/vtgate/misc_test.go index d863af2a47..ebd5eb7b14 100644 --- a/go/test/endtoend/vtgate/misc_test.go +++ b/go/test/endtoend/vtgate/misc_test.go @@ -669,13 +669,13 @@ func TestVSchemaTrackerInitAndUpdate(t *testing.T) { require.NoError(t, err) defer conn.Close() - assertMatches(t, conn, "SHOW VSCHEMA TABLES", `[[VARCHAR("aggr_test")] [VARCHAR("dual")] [VARCHAR("t1")] [VARCHAR("t1_id2_idx")] [VARCHAR("t2")] [VARCHAR("t2_id4_idx")] [VARCHAR("t3")] [VARCHAR("t3_id7_idx")] [VARCHAR("t4")] [VARCHAR("t4_id2_idx")] [VARCHAR("t5_null_vindex")] [VARCHAR("t6")] [VARCHAR("t6_id2_idx")] [VARCHAR("t7_fk")] [VARCHAR("t7_xxhash")] [VARCHAR("t7_xxhash_idx")] [VARCHAR("t8")] [VARCHAR("vstream_test")]]`) + assertMatches(t, conn, "SHOW VSCHEMA TABLES", `[[VARCHAR("aggr_test")] [VARCHAR("dual")] [VARCHAR("t1")] [VARCHAR("t1_id2_idx")] [VARCHAR("t2")] [VARCHAR("t2_id4_idx")] [VARCHAR("t3")] [VARCHAR("t3_id7_idx")] [VARCHAR("t4")] [VARCHAR("t4_id2_idx")] [VARCHAR("t5_null_vindex")] [VARCHAR("t6")] [VARCHAR("t6_id2_idx")] [VARCHAR("t7_fk")] [VARCHAR("t7_xxhash")] [VARCHAR("t7_xxhash_idx")] [VARCHAR("t8")] [VARCHAR("t9")] [VARCHAR("vstream_test")]]`) // Init _ = exec(t, conn, "create table test_sc (id bigint primary key)") assertMatchesWithTimeout(t, conn, "SHOW VSCHEMA TABLES", - `[[VARCHAR("aggr_test")] [VARCHAR("dual")] [VARCHAR("t1")] [VARCHAR("t1_id2_idx")] [VARCHAR("t2")] [VARCHAR("t2_id4_idx")] [VARCHAR("t3")] [VARCHAR("t3_id7_idx")] [VARCHAR("t4")] [VARCHAR("t4_id2_idx")] [VARCHAR("t5_null_vindex")] [VARCHAR("t6")] [VARCHAR("t6_id2_idx")] [VARCHAR("t7_fk")] [VARCHAR("t7_xxhash")] [VARCHAR("t7_xxhash_idx")] [VARCHAR("t8")] [VARCHAR("test_sc")] [VARCHAR("vstream_test")]]`, + `[[VARCHAR("aggr_test")] [VARCHAR("dual")] [VARCHAR("t1")] [VARCHAR("t1_id2_idx")] [VARCHAR("t2")] [VARCHAR("t2_id4_idx")] [VARCHAR("t3")] [VARCHAR("t3_id7_idx")] [VARCHAR("t4")] [VARCHAR("t4_id2_idx")] [VARCHAR("t5_null_vindex")] [VARCHAR("t6")] [VARCHAR("t6_id2_idx")] [VARCHAR("t7_fk")] [VARCHAR("t7_xxhash")] [VARCHAR("t7_xxhash_idx")] [VARCHAR("t8")] [VARCHAR("t9")] [VARCHAR("test_sc")] [VARCHAR("vstream_test")]]`, 100*time.Millisecond, 3*time.Second, "test_sc not in vschema tables") @@ -684,7 +684,7 @@ func TestVSchemaTrackerInitAndUpdate(t *testing.T) { _ = exec(t, conn, "create table test_sc1 (id bigint primary key)") assertMatchesWithTimeout(t, conn, "SHOW VSCHEMA TABLES", - `[[VARCHAR("aggr_test")] [VARCHAR("dual")] [VARCHAR("t1")] [VARCHAR("t1_id2_idx")] [VARCHAR("t2")] [VARCHAR("t2_id4_idx")] [VARCHAR("t3")] [VARCHAR("t3_id7_idx")] [VARCHAR("t4")] [VARCHAR("t4_id2_idx")] [VARCHAR("t5_null_vindex")] [VARCHAR("t6")] [VARCHAR("t6_id2_idx")] [VARCHAR("t7_fk")] [VARCHAR("t7_xxhash")] [VARCHAR("t7_xxhash_idx")] [VARCHAR("t8")] [VARCHAR("test_sc")] [VARCHAR("test_sc1")] [VARCHAR("vstream_test")]]`, + `[[VARCHAR("aggr_test")] [VARCHAR("dual")] [VARCHAR("t1")] [VARCHAR("t1_id2_idx")] [VARCHAR("t2")] [VARCHAR("t2_id4_idx")] [VARCHAR("t3")] [VARCHAR("t3_id7_idx")] [VARCHAR("t4")] [VARCHAR("t4_id2_idx")] [VARCHAR("t5_null_vindex")] [VARCHAR("t6")] [VARCHAR("t6_id2_idx")] [VARCHAR("t7_fk")] [VARCHAR("t7_xxhash")] [VARCHAR("t7_xxhash_idx")] [VARCHAR("t8")] [VARCHAR("t9")] [VARCHAR("test_sc")] [VARCHAR("test_sc1")] [VARCHAR("vstream_test")]]`, 100*time.Millisecond, 3*time.Second, "test_sc1 not in vschema tables") From 4213c523228958611d4d1edbf6b0f6e8d528782a Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Thu, 10 Jun 2021 13:39:16 +0530 Subject: [PATCH 53/64] fixed error messages and reorganized tests wrt schema tracker Signed-off-by: GuptaManan100 --- go/mysql/sql_error.go | 1 + go/test/endtoend/vtgate/misc_test.go | 84 ----- .../schematracker/schematracker_test.go | 107 ------- .../schematracker/sharded/st_sharded_test.go | 298 ++++++++++++++++++ .../unsharded/st_unsharded_test.go | 159 ++++++++++ go/vt/vterrors/constants.go | 6 + go/vt/vterrors/state.go | 1 + go/vt/vtgate/engine/insert.go | 2 +- go/vt/vtgate/planbuilder/dml.go | 2 +- 9 files changed, 467 insertions(+), 193 deletions(-) delete mode 100644 go/test/endtoend/vtgate/schematracker/schematracker_test.go create mode 100644 go/test/endtoend/vtgate/schematracker/sharded/st_sharded_test.go create mode 100644 go/test/endtoend/vtgate/schematracker/unsharded/st_unsharded_test.go diff --git a/go/mysql/sql_error.go b/go/mysql/sql_error.go index b2d12af0c4..369e7a6f0e 100644 --- a/go/mysql/sql_error.go +++ b/go/mysql/sql_error.go @@ -190,6 +190,7 @@ var stateToMysqlCode = map[vterrors.State]struct { vterrors.WrongValueForVar: {num: ERWrongValueForVar, state: SSClientError}, vterrors.ServerNotAvailable: {num: ERServerIsntAvailable, state: SSNetError}, vterrors.CantDoThisInTransaction: {num: ERCantDoThisDuringAnTransaction, state: SSCantDoThisDuringAnTransaction}, + vterrors.RequiresPrimaryKey: {num: ERRequiresPrimaryKey, state: SSClientError}, vterrors.NoSuchSession: {num: ERUnknownComError, state: SSNetError}, } diff --git a/go/test/endtoend/vtgate/misc_test.go b/go/test/endtoend/vtgate/misc_test.go index ebd5eb7b14..d4c3d5a794 100644 --- a/go/test/endtoend/vtgate/misc_test.go +++ b/go/test/endtoend/vtgate/misc_test.go @@ -20,7 +20,6 @@ import ( "context" "fmt" "testing" - "time" "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/assert" @@ -651,73 +650,6 @@ func TestQueryAndSubQWithLimit(t *testing.T) { assert.Equal(t, 10, len(result.Rows)) } -func TestSchemaTracker(t *testing.T) { - defer cluster.PanicHandler(t) - ctx := context.Background() - conn, err := mysql.Connect(ctx, &vtParams) - require.NoError(t, err) - defer conn.Close() - // this query only works if we know which table the testId belongs to. The vschema does not contain - // this info, so we are testing that the schema tracker has added column info to the vschema - _, err = conn.ExecuteFetch(`select testId from t8 join t2`, 1000, true) - require.NoError(t, err) -} - -func TestVSchemaTrackerInitAndUpdate(t *testing.T) { - ctx := context.Background() - conn, err := mysql.Connect(ctx, &vtParams) - require.NoError(t, err) - defer conn.Close() - - assertMatches(t, conn, "SHOW VSCHEMA TABLES", `[[VARCHAR("aggr_test")] [VARCHAR("dual")] [VARCHAR("t1")] [VARCHAR("t1_id2_idx")] [VARCHAR("t2")] [VARCHAR("t2_id4_idx")] [VARCHAR("t3")] [VARCHAR("t3_id7_idx")] [VARCHAR("t4")] [VARCHAR("t4_id2_idx")] [VARCHAR("t5_null_vindex")] [VARCHAR("t6")] [VARCHAR("t6_id2_idx")] [VARCHAR("t7_fk")] [VARCHAR("t7_xxhash")] [VARCHAR("t7_xxhash_idx")] [VARCHAR("t8")] [VARCHAR("t9")] [VARCHAR("vstream_test")]]`) - - // Init - _ = exec(t, conn, "create table test_sc (id bigint primary key)") - assertMatchesWithTimeout(t, conn, - "SHOW VSCHEMA TABLES", - `[[VARCHAR("aggr_test")] [VARCHAR("dual")] [VARCHAR("t1")] [VARCHAR("t1_id2_idx")] [VARCHAR("t2")] [VARCHAR("t2_id4_idx")] [VARCHAR("t3")] [VARCHAR("t3_id7_idx")] [VARCHAR("t4")] [VARCHAR("t4_id2_idx")] [VARCHAR("t5_null_vindex")] [VARCHAR("t6")] [VARCHAR("t6_id2_idx")] [VARCHAR("t7_fk")] [VARCHAR("t7_xxhash")] [VARCHAR("t7_xxhash_idx")] [VARCHAR("t8")] [VARCHAR("t9")] [VARCHAR("test_sc")] [VARCHAR("vstream_test")]]`, - 100*time.Millisecond, - 3*time.Second, - "test_sc not in vschema tables") - - // Tables Update via health check. - _ = exec(t, conn, "create table test_sc1 (id bigint primary key)") - assertMatchesWithTimeout(t, conn, - "SHOW VSCHEMA TABLES", - `[[VARCHAR("aggr_test")] [VARCHAR("dual")] [VARCHAR("t1")] [VARCHAR("t1_id2_idx")] [VARCHAR("t2")] [VARCHAR("t2_id4_idx")] [VARCHAR("t3")] [VARCHAR("t3_id7_idx")] [VARCHAR("t4")] [VARCHAR("t4_id2_idx")] [VARCHAR("t5_null_vindex")] [VARCHAR("t6")] [VARCHAR("t6_id2_idx")] [VARCHAR("t7_fk")] [VARCHAR("t7_xxhash")] [VARCHAR("t7_xxhash_idx")] [VARCHAR("t8")] [VARCHAR("t9")] [VARCHAR("test_sc")] [VARCHAR("test_sc1")] [VARCHAR("vstream_test")]]`, - 100*time.Millisecond, - 3*time.Second, - "test_sc1 not in vschema tables") -} - -func TestVSchemaTrackedForNewTables(t *testing.T) { - ctx := context.Background() - conn, err := mysql.Connect(ctx, &vtParams) - require.NoError(t, err) - defer conn.Close() - - // create a new table which is not part of the VSchema - exec(t, conn, `create table new_table_tracked(id bigint, name varchar(100), primary key(id)) Engine=InnoDB`) - - // wait for vttablet's schema reload interval to pass - time.Sleep(5 * time.Second) - - // check if the new table is part of the schema - assertMatches(t, conn, "SHOW VSCHEMA TABLES", `[[VARCHAR("aggr_test")] [VARCHAR("dual")] [VARCHAR("new_table_tracked")] [VARCHAR("t1")] [VARCHAR("t1_id2_idx")] [VARCHAR("t2")] [VARCHAR("t2_id4_idx")] [VARCHAR("t3")] [VARCHAR("t3_id7_idx")] [VARCHAR("t4")] [VARCHAR("t4_id2_idx")] [VARCHAR("t5_null_vindex")] [VARCHAR("t6")] [VARCHAR("t6_id2_idx")] [VARCHAR("t7_fk")] [VARCHAR("t7_xxhash")] [VARCHAR("t7_xxhash_idx")] [VARCHAR("t8")] [VARCHAR("t9")] [VARCHAR("vstream_test")]]`) - - // DML on new table - assertMatches(t, conn, "select id from new_table_tracked", `[]`) // select - exec(t, conn, `insert into new_table_tracked(id) values(0),(1)`) // insert initial data - assertMatches(t, conn, "select id from new_table_tracked", `[[INT64(0)] [INT64(1)]]`) // select - assertMatches(t, conn, "select id from new_table_tracked where id = 1 ", `[[INT64(1)]]`) // select with WHERE clause - - exec(t, conn, `update new_table_tracked set name = "newName1" where id = 1`) // update - assertMatches(t, conn, "select name from new_table_tracked where id = 1 ", `[[VARCHAR("newName1")]]`) - - exec(t, conn, "delete from new_table_tracked where id = 0") // delete - assertMatches(t, conn, "select id from new_table_tracked", `[[INT64(1)]]`) -} - func assertMatches(t *testing.T, conn *mysql.Conn, query, expected string) { t.Helper() qr := exec(t, conn, query) @@ -728,22 +660,6 @@ func assertMatches(t *testing.T, conn *mysql.Conn, query, expected string) { } } -func assertMatchesWithTimeout(t *testing.T, conn *mysql.Conn, query, expected string, r time.Duration, d time.Duration, failureMsg string) { - t.Helper() - timeout := time.After(d) - diff := "actual and expectation does not match" - for len(diff) > 0 { - select { - case <-timeout: - require.Fail(t, failureMsg, diff) - case <-time.After(r): - qr := exec(t, conn, `SHOW VSCHEMA TABLES`) - diff = cmp.Diff(expected, - fmt.Sprintf("%v", qr.Rows)) - } - - } -} func assertMatchesNoOrder(t *testing.T, conn *mysql.Conn, query, expected string) { t.Helper() qr := exec(t, conn, query) diff --git a/go/test/endtoend/vtgate/schematracker/schematracker_test.go b/go/test/endtoend/vtgate/schematracker/schematracker_test.go deleted file mode 100644 index dbc9dab3de..0000000000 --- a/go/test/endtoend/vtgate/schematracker/schematracker_test.go +++ /dev/null @@ -1,107 +0,0 @@ -/* -Copyright 2021 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package schematracker - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "vitess.io/vitess/go/mysql" - "vitess.io/vitess/go/sqltypes" - "vitess.io/vitess/go/test/endtoend/cluster" -) - -var ( - hostname = "localhost" - keyspaceName = "ks" - cell = "zone1" - sqlSchema = ` - create table main ( - id bigint, - val varchar(128), - primary key(id) - ) Engine=InnoDB; -` -) - -func TestNewUnshardedTable(t *testing.T) { - defer cluster.PanicHandler(t) - var err error - - // initialize our cluster - clusterInstance := cluster.NewCluster(cell, hostname) - defer clusterInstance.Teardown() - - // Start topo server - err = clusterInstance.StartTopo() - require.NoError(t, err) - - // create keyspace - keyspace := &cluster.Keyspace{ - Name: keyspaceName, - SchemaSQL: sqlSchema, - } - - // enabling and setting the schema reload time to one second - clusterInstance.VtTabletExtraArgs = []string{"-queryserver-config-schema-change-signal", "-queryserver-config-schema-change-signal-interval", "0.1"} - err = clusterInstance.StartUnshardedKeyspace(*keyspace, 1, false) - require.NoError(t, err) - - // Start vtgate with the schema_change_signal flag - clusterInstance.VtGateExtraArgs = []string{"-schema_change_signal"} - err = clusterInstance.StartVtgate() - require.NoError(t, err) - - // create a sql connection - ctx := context.Background() - conn, err := mysql.Connect(ctx, &mysql.ConnParams{ - Host: clusterInstance.Hostname, - Port: clusterInstance.VtgateMySQLPort, - }) - require.NoError(t, err) - defer conn.Close() - - // ensuring our initial table "main" is in the schema - qr := exec(t, conn, "SHOW VSCHEMA TABLES") - got := fmt.Sprintf("%v", qr.Rows) - want := `[[VARCHAR("dual")] [VARCHAR("main")]]` - require.Equal(t, want, got) - - // create a new table which is not part of the VSchema - exec(t, conn, `create table new_table_tracked(id bigint, name varchar(100), primary key(id)) Engine=InnoDB`) - - // waiting for the vttablet's schema_reload interval to kick in - time.Sleep(5 * time.Second) - - // ensuring our new table is in the schema - qr = exec(t, conn, "SHOW VSCHEMA TABLES") - got = fmt.Sprintf("%v", qr.Rows) - want = `[[VARCHAR("dual")] [VARCHAR("main")] [VARCHAR("new_table_tracked")]]` - assert.Equal(t, want, got) -} - -func exec(t *testing.T, conn *mysql.Conn, query string) *sqltypes.Result { - t.Helper() - qr, err := conn.ExecuteFetch(query, 1000, true) - require.NoError(t, err, "for query: "+query) - return qr -} diff --git a/go/test/endtoend/vtgate/schematracker/sharded/st_sharded_test.go b/go/test/endtoend/vtgate/schematracker/sharded/st_sharded_test.go new file mode 100644 index 0000000000..a41616bb66 --- /dev/null +++ b/go/test/endtoend/vtgate/schematracker/sharded/st_sharded_test.go @@ -0,0 +1,298 @@ +/* +Copyright 2021 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package sharded + +import ( + "context" + "flag" + "fmt" + "os" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + + "vitess.io/vitess/go/test/utils" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/test/endtoend/cluster" +) + +var ( + clusterInstance *cluster.LocalProcessCluster + vtParams mysql.ConnParams + KeyspaceName = "ks" + Cell = "test" + SchemaSQL = ` +create table t2( + id3 bigint, + id4 bigint, + primary key(id3) +) Engine=InnoDB; + +create table t2_id4_idx( + id bigint not null auto_increment, + id4 bigint, + id3 bigint, + primary key(id), + key idx_id4(id4) +) Engine=InnoDB; + +create table t8( + id8 bigint, + testId bigint, + primary key(id8) +) Engine=InnoDB; +` + + VSchema = ` +{ + "sharded": true, + "vindexes": { + "unicode_loose_xxhash" : { + "type": "unicode_loose_xxhash" + }, + "unicode_loose_md5" : { + "type": "unicode_loose_md5" + }, + "hash": { + "type": "hash" + }, + "xxhash": { + "type": "xxhash" + }, + "t2_id4_idx": { + "type": "lookup_hash", + "params": { + "table": "t2_id4_idx", + "from": "id4", + "to": "id3", + "autocommit": "true" + }, + "owner": "t2" + } + }, + "tables": { + "t2": { + "column_vindexes": [ + { + "column": "id3", + "name": "hash" + }, + { + "column": "id4", + "name": "t2_id4_idx" + } + ] + }, + "t2_id4_idx": { + "column_vindexes": [ + { + "column": "id4", + "name": "hash" + } + ] + }, + "t8": { + "column_vindexes": [ + { + "column": "id8", + "name": "hash" + } + ] + } + } +}` +) + +func TestMain(m *testing.M) { + defer cluster.PanicHandler(nil) + flag.Parse() + + exitCode := func() int { + clusterInstance = cluster.NewCluster(Cell, "localhost") + defer clusterInstance.Teardown() + + // Start topo server + err := clusterInstance.StartTopo() + if err != nil { + return 1 + } + + // Start keyspace + keyspace := &cluster.Keyspace{ + Name: KeyspaceName, + SchemaSQL: SchemaSQL, + VSchema: VSchema, + } + clusterInstance.VtGateExtraArgs = []string{"-schema_change_signal", "-vschema_ddl_authorized_users", "%"} + clusterInstance.VtTabletExtraArgs = []string{"-queryserver-config-schema-change-signal", "-queryserver-config-schema-change-signal-interval", "0.1"} + err = clusterInstance.StartKeyspace(*keyspace, []string{"-80", "80-"}, 1, true) + if err != nil { + return 1 + } + + // Start vtgate + err = clusterInstance.StartVtgate() + if err != nil { + return 1 + } + + vtParams = mysql.ConnParams{ + Host: clusterInstance.Hostname, + Port: clusterInstance.VtgateMySQLPort, + } + return m.Run() + }() + os.Exit(exitCode) +} + +func TestSchemaTracker(t *testing.T) { + defer cluster.PanicHandler(t) + ctx := context.Background() + conn, err := mysql.Connect(ctx, &vtParams) + require.NoError(t, err) + defer conn.Close() + // this query only works if we know which table the testId belongs to. The vschema does not contain + // this info, so we are testing that the schema tracker has added column info to the vschema + _, err = conn.ExecuteFetch(`select testId from t8 join t2`, 1000, true) + require.NoError(t, err) +} + +func TestVSchemaTrackerInitAndUpdate(t *testing.T) { + ctx := context.Background() + conn, err := mysql.Connect(ctx, &vtParams) + require.NoError(t, err) + defer conn.Close() + + assertMatches(t, conn, "SHOW VSCHEMA TABLES", `[[VARCHAR("dual")] [VARCHAR("t2")] [VARCHAR("t2_id4_idx")] [VARCHAR("t8")]]`) + + // Init + _ = exec(t, conn, "create table test_sc (id bigint primary key)") + assertMatchesWithTimeout(t, conn, + "SHOW VSCHEMA TABLES", + `[[VARCHAR("dual")] [VARCHAR("t2")] [VARCHAR("t2_id4_idx")] [VARCHAR("t8")] [VARCHAR("test_sc")]]`, + 100*time.Millisecond, + 3*time.Second, + "test_sc not in vschema tables") + + // Tables Update via health check. + _ = exec(t, conn, "create table test_sc1 (id bigint primary key)") + assertMatchesWithTimeout(t, conn, + "SHOW VSCHEMA TABLES", + `[[VARCHAR("dual")] [VARCHAR("t2")] [VARCHAR("t2_id4_idx")] [VARCHAR("t8")] [VARCHAR("test_sc")] [VARCHAR("test_sc1")]]`, + 100*time.Millisecond, + 3*time.Second, + "test_sc1 not in vschema tables") + + _ = exec(t, conn, "drop table test_sc, test_sc1") + assertMatchesWithTimeout(t, conn, + "SHOW VSCHEMA TABLES", + `[[VARCHAR("dual")] [VARCHAR("t2")] [VARCHAR("t2_id4_idx")] [VARCHAR("t8")]]`, + 100*time.Millisecond, + 3*time.Second, + "test_sc and test_sc_1 should not be in vschema tables") + +} + +func TestVSchemaTrackedForNewTables(t *testing.T) { + ctx := context.Background() + conn, err := mysql.Connect(ctx, &vtParams) + require.NoError(t, err) + defer conn.Close() + + // create a new table which is not part of the VSchema + exec(t, conn, `create table new_table_tracked(id bigint, name varchar(100), primary key(id)) Engine=InnoDB`) + + // wait for vttablet's schema reload interval to pass + assertMatchesWithTimeout(t, conn, + "SHOW VSCHEMA TABLES", + `[[VARCHAR("dual")] [VARCHAR("new_table_tracked")] [VARCHAR("t2")] [VARCHAR("t2_id4_idx")] [VARCHAR("t8")]]`, + 100*time.Millisecond, + 3*time.Second, + "test_sc not in vschema tables") + + assertMatches(t, conn, "select id from new_table_tracked", `[]`) // select + assertMatches(t, conn, "select id from new_table_tracked where id = 5", `[]`) // select + // DML on new table + // insert initial data ,update and delete will fail since we have not added a primary vindex + errorMessage := "table 'new_table_tracked' does not have a primary vindex (errno 1173) (sqlstate 42000)" + assertError(t, conn, `insert into new_table_tracked(id) values(0),(1)`, errorMessage) + assertError(t, conn, `update new_table_tracked set name = "newName1"`, errorMessage) + assertError(t, conn, "delete from new_table_tracked", errorMessage) + + exec(t, conn, `select name from new_table_tracked join t8`) + + // add a primary vindex for the table + exec(t, conn, "alter vschema on ks.new_table_tracked add vindex hash(id) using hash") + time.Sleep(1 * time.Second) + exec(t, conn, `insert into new_table_tracked(id) values(0),(1)`) + exec(t, conn, `insert into t8(id8) values(2)`) + defer exec(t, conn, `delete from t8`) + assertMatchesNoOrder(t, conn, `select id from new_table_tracked join t8`, `[[INT64(0)] [INT64(1)]]`) +} + +func assertMatches(t *testing.T, conn *mysql.Conn, query, expected string) { + t.Helper() + qr := exec(t, conn, query) + got := fmt.Sprintf("%v", qr.Rows) + diff := cmp.Diff(expected, got) + if diff != "" { + t.Errorf("Query: %s (-want +got):\n%s", query, diff) + } +} + +func assertMatchesWithTimeout(t *testing.T, conn *mysql.Conn, query, expected string, r time.Duration, d time.Duration, failureMsg string) { + t.Helper() + timeout := time.After(d) + diff := "actual and expectation does not match" + for len(diff) > 0 { + select { + case <-timeout: + require.Fail(t, failureMsg, diff) + case <-time.After(r): + qr := exec(t, conn, query) + diff = cmp.Diff(expected, + fmt.Sprintf("%v", qr.Rows)) + } + + } +} +func assertMatchesNoOrder(t *testing.T, conn *mysql.Conn, query, expected string) { + t.Helper() + qr := exec(t, conn, query) + actual := fmt.Sprintf("%v", qr.Rows) + assert.Equal(t, utils.SortString(expected), utils.SortString(actual), "for query: [%s] expected \n%s \nbut actual \n%s", query, expected, actual) +} + +func assertError(t *testing.T, conn *mysql.Conn, query, errorMessage string) { + t.Helper() + _, err := conn.ExecuteFetch(query, 1000, true) + require.Error(t, err) + assert.Contains(t, err.Error(), errorMessage) +} + +func exec(t *testing.T, conn *mysql.Conn, query string) *sqltypes.Result { + t.Helper() + qr, err := conn.ExecuteFetch(query, 1000, true) + require.NoError(t, err, "for query: "+query) + return qr +} diff --git a/go/test/endtoend/vtgate/schematracker/unsharded/st_unsharded_test.go b/go/test/endtoend/vtgate/schematracker/unsharded/st_unsharded_test.go new file mode 100644 index 0000000000..a479d85959 --- /dev/null +++ b/go/test/endtoend/vtgate/schematracker/unsharded/st_unsharded_test.go @@ -0,0 +1,159 @@ +/* +Copyright 2021 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package unsharded + +import ( + "context" + "flag" + "fmt" + "os" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/test/endtoend/cluster" +) + +var ( + clusterInstance *cluster.LocalProcessCluster + vtParams mysql.ConnParams + keyspaceName = "ks" + cell = "zone1" + sqlSchema = ` + create table main ( + id bigint, + val varchar(128), + primary key(id) + ) Engine=InnoDB; +` +) + +func TestMain(m *testing.M) { + defer cluster.PanicHandler(nil) + flag.Parse() + + exitCode := func() int { + clusterInstance = cluster.NewCluster(cell, "localhost") + defer clusterInstance.Teardown() + + // Start topo server + err := clusterInstance.StartTopo() + if err != nil { + return 1 + } + + // Start keyspace + keyspace := &cluster.Keyspace{ + Name: keyspaceName, + SchemaSQL: sqlSchema, + } + clusterInstance.VtTabletExtraArgs = []string{"-queryserver-config-schema-change-signal", "-queryserver-config-schema-change-signal-interval", "0.1"} + err = clusterInstance.StartUnshardedKeyspace(*keyspace, 0, false) + if err != nil { + return 1 + } + + // Start vtgate + clusterInstance.VtGateExtraArgs = []string{"-schema_change_signal", "-vschema_ddl_authorized_users", "%"} + err = clusterInstance.StartVtgate() + if err != nil { + return 1 + } + + vtParams = mysql.ConnParams{ + Host: clusterInstance.Hostname, + Port: clusterInstance.VtgateMySQLPort, + } + return m.Run() + }() + os.Exit(exitCode) +} + +func TestNewUnshardedTable(t *testing.T) { + defer cluster.PanicHandler(t) + + // create a sql connection + ctx := context.Background() + conn, err := mysql.Connect(ctx, &vtParams) + require.NoError(t, err) + defer conn.Close() + + // ensuring our initial table "main" is in the schema + qr := exec(t, conn, "SHOW VSCHEMA TABLES") + got := fmt.Sprintf("%v", qr.Rows) + want := `[[VARCHAR("dual")] [VARCHAR("main")]]` + require.Equal(t, want, got) + + // create a new table which is not part of the VSchema + exec(t, conn, `create table new_table_tracked(id bigint, name varchar(100), primary key(id)) Engine=InnoDB`) + defer exec(t, conn, `drop table new_table_tracked`) + + // waiting for the vttablet's schema_reload interval to kick in + assertMatchesWithTimeout(t, conn, + "SHOW VSCHEMA TABLES", + `[[VARCHAR("dual")] [VARCHAR("main")] [VARCHAR("new_table_tracked")]]`, + 100*time.Millisecond, + 3*time.Second, + "new_table_tracked not in vschema tables") + + assertMatches(t, conn, "select id from new_table_tracked", `[]`) // select + assertMatches(t, conn, "select id from new_table_tracked where id = 5", `[]`) // select + // DML on new table + // insert initial data ,update and delete for the new table + exec(t, conn, `insert into new_table_tracked(id) values(0),(1)`) + exec(t, conn, `update new_table_tracked set name = "newName1"`) + exec(t, conn, "delete from new_table_tracked where id = 0") + assertMatches(t, conn, `select * from new_table_tracked`, `[[INT64(1) VARCHAR("newName1")]]`) +} + +func assertMatches(t *testing.T, conn *mysql.Conn, query, expected string) { + t.Helper() + qr := exec(t, conn, query) + got := fmt.Sprintf("%v", qr.Rows) + diff := cmp.Diff(expected, got) + if diff != "" { + t.Errorf("Query: %s (-want +got):\n%s", query, diff) + } +} + +func assertMatchesWithTimeout(t *testing.T, conn *mysql.Conn, query, expected string, r time.Duration, d time.Duration, failureMsg string) { + t.Helper() + timeout := time.After(d) + diff := "actual and expectation does not match" + for len(diff) > 0 { + select { + case <-timeout: + require.Fail(t, failureMsg, diff) + case <-time.After(r): + qr := exec(t, conn, query) + diff = cmp.Diff(expected, + fmt.Sprintf("%v", qr.Rows)) + } + + } +} + +func exec(t *testing.T, conn *mysql.Conn, query string) *sqltypes.Result { + t.Helper() + qr, err := conn.ExecuteFetch(query, 1000, true) + require.NoError(t, err, "for query: "+query) + return qr +} diff --git a/go/vt/vterrors/constants.go b/go/vt/vterrors/constants.go index d66a97464d..2fcbfda13d 100644 --- a/go/vt/vterrors/constants.go +++ b/go/vt/vterrors/constants.go @@ -32,3 +32,9 @@ const WrongTablet = "wrong tablet type" // RxWrongTablet regex for invalid tablet type error var RxWrongTablet = regexp.MustCompile("(wrong|invalid) tablet type") + +// Constants for error messages +const ( + // PrimaryVindexNotSet is the error message to be used when there is no primary vindex found on a table + PrimaryVindexNotSet = "table '%s' does not have a primary vindex" +) diff --git a/go/vt/vterrors/state.go b/go/vt/vterrors/state.go index 8b4e8b9b4a..29575de8dd 100644 --- a/go/vt/vterrors/state.go +++ b/go/vt/vterrors/state.go @@ -43,6 +43,7 @@ const ( InnodbReadOnly WrongNumberOfColumnsInSelect CantDoThisInTransaction + RequiresPrimaryKey // not found BadDb diff --git a/go/vt/vtgate/engine/insert.go b/go/vt/vtgate/engine/insert.go index 4774cad2a6..00188b9921 100644 --- a/go/vt/vtgate/engine/insert.go +++ b/go/vt/vtgate/engine/insert.go @@ -395,7 +395,7 @@ func (ins *Insert) getInsertShardedRoute(vcursor VCursor, bindVars map[string]*q // results in an error. For 'ignore' type inserts, the keyspace // id is returned as nil, which is used later to drop the corresponding rows. if len(vindexRowsValues) == 0 || len(ins.Table.ColumnVindexes) == 0 { - return nil, nil, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "[BUG] table without a primary vindex is not expectedd") + return nil, nil, vterrors.NewErrorf(vtrpcpb.Code_FAILED_PRECONDITION, vterrors.RequiresPrimaryKey, vterrors.PrimaryVindexNotSet, ins.Table.Name) } keyspaceIDs, err := ins.processPrimary(vcursor, vindexRowsValues[0], ins.Table.ColumnVindexes[0]) if err != nil { diff --git a/go/vt/vtgate/planbuilder/dml.go b/go/vt/vtgate/planbuilder/dml.go index 42530579fa..cbd1654273 100644 --- a/go/vt/vtgate/planbuilder/dml.go +++ b/go/vt/vtgate/planbuilder/dml.go @@ -56,7 +56,7 @@ func getDMLRouting(where *sqlparser.Where, table *vindexes.Table) (engine.DMLOpc } } if ksidVindex == nil { - return engine.Scatter, nil, "", nil, nil, vterrors.New(vtrpcpb.Code_INTERNAL, "table without a primary vindex is not expected") + return engine.Scatter, nil, "", nil, nil, vterrors.NewErrorf(vtrpcpb.Code_FAILED_PRECONDITION, vterrors.RequiresPrimaryKey, vterrors.PrimaryVindexNotSet, table.Name) } return engine.Scatter, ksidVindex, ksidCol, nil, nil, nil } From 6087205c7e4b72d44b694b58624b55df3e49b09e Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Thu, 10 Jun 2021 13:44:16 +0530 Subject: [PATCH 54/64] update names of tests Signed-off-by: GuptaManan100 --- go/test/endtoend/vtgate/main_test.go | 14 -------------- .../schematracker/sharded/st_sharded_test.go | 7 ++++--- .../schematracker/unsharded/st_unsharded_test.go | 11 ++++++++++- 3 files changed, 14 insertions(+), 18 deletions(-) diff --git a/go/test/endtoend/vtgate/main_test.go b/go/test/endtoend/vtgate/main_test.go index dd17f6192f..a8b8baea1d 100644 --- a/go/test/endtoend/vtgate/main_test.go +++ b/go/test/endtoend/vtgate/main_test.go @@ -142,12 +142,6 @@ create table t8( testId bigint, primary key(id8) ) Engine=InnoDB; - -create table t9( - id9 bigint, - testId bigint, - primary key(id9) -) Engine=InnoDB; ` VSchema = ` @@ -390,14 +384,6 @@ create table t9( "name": "hash" } ] - }, - "t9": { - "column_vindexes": [ - { - "column": "id9", - "name": "hash" - } - ] } } }` diff --git a/go/test/endtoend/vtgate/schematracker/sharded/st_sharded_test.go b/go/test/endtoend/vtgate/schematracker/sharded/st_sharded_test.go index a41616bb66..0890157129 100644 --- a/go/test/endtoend/vtgate/schematracker/sharded/st_sharded_test.go +++ b/go/test/endtoend/vtgate/schematracker/sharded/st_sharded_test.go @@ -165,7 +165,7 @@ func TestMain(m *testing.M) { os.Exit(exitCode) } -func TestSchemaTracker(t *testing.T) { +func TestAmbiguousColumnJoin(t *testing.T) { defer cluster.PanicHandler(t) ctx := context.Background() conn, err := mysql.Connect(ctx, &vtParams) @@ -177,7 +177,7 @@ func TestSchemaTracker(t *testing.T) { require.NoError(t, err) } -func TestVSchemaTrackerInitAndUpdate(t *testing.T) { +func TestInitAndUpdate(t *testing.T) { ctx := context.Background() conn, err := mysql.Connect(ctx, &vtParams) require.NoError(t, err) @@ -213,7 +213,7 @@ func TestVSchemaTrackerInitAndUpdate(t *testing.T) { } -func TestVSchemaTrackedForNewTables(t *testing.T) { +func TestDMLOnNewTable(t *testing.T) { ctx := context.Background() conn, err := mysql.Connect(ctx, &vtParams) require.NoError(t, err) @@ -276,6 +276,7 @@ func assertMatchesWithTimeout(t *testing.T, conn *mysql.Conn, query, expected st } } + func assertMatchesNoOrder(t *testing.T, conn *mysql.Conn, query, expected string) { t.Helper() qr := exec(t, conn, query) diff --git a/go/test/endtoend/vtgate/schematracker/unsharded/st_unsharded_test.go b/go/test/endtoend/vtgate/schematracker/unsharded/st_unsharded_test.go index a479d85959..e4ced16670 100644 --- a/go/test/endtoend/vtgate/schematracker/unsharded/st_unsharded_test.go +++ b/go/test/endtoend/vtgate/schematracker/unsharded/st_unsharded_test.go @@ -104,7 +104,6 @@ func TestNewUnshardedTable(t *testing.T) { // create a new table which is not part of the VSchema exec(t, conn, `create table new_table_tracked(id bigint, name varchar(100), primary key(id)) Engine=InnoDB`) - defer exec(t, conn, `drop table new_table_tracked`) // waiting for the vttablet's schema_reload interval to kick in assertMatchesWithTimeout(t, conn, @@ -122,6 +121,16 @@ func TestNewUnshardedTable(t *testing.T) { exec(t, conn, `update new_table_tracked set name = "newName1"`) exec(t, conn, "delete from new_table_tracked where id = 0") assertMatches(t, conn, `select * from new_table_tracked`, `[[INT64(1) VARCHAR("newName1")]]`) + + exec(t, conn, `drop table new_table_tracked`) + + // waiting for the vttablet's schema_reload interval to kick in + assertMatchesWithTimeout(t, conn, + "SHOW VSCHEMA TABLES", + `[[VARCHAR("dual")] [VARCHAR("main")]]`, + 100*time.Millisecond, + 3*time.Second, + "new_table_tracked not in vschema tables") } func assertMatches(t *testing.T, conn *mysql.Conn, query, expected string) { From 61245df267a2242bbaca3b942e4c4ad8e39f2fe8 Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Thu, 10 Jun 2021 13:54:06 +0530 Subject: [PATCH 55/64] removed unnecessary channel close Signed-off-by: GuptaManan100 --- go/vt/vtgate/schema/tracker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/vtgate/schema/tracker.go b/go/vt/vtgate/schema/tracker.go index f819bbcd36..eeb13992f1 100644 --- a/go/vt/vtgate/schema/tracker.go +++ b/go/vt/vtgate/schema/tracker.go @@ -93,7 +93,7 @@ func (t *Tracker) Start() { ksUpdater := t.getKeyspaceUpdateController(th) ksUpdater.add(th) case <-ctx.Done(): - close(t.ch) + // closing of the channel happens outside the scope of the tracker. It is the responsibility of the one who created this tracker. return } } From 03c1a5b298ee9cbfc727f6cdb36278166473548b Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Thu, 10 Jun 2021 15:16:55 +0530 Subject: [PATCH 56/64] fixed test configuration Signed-off-by: GuptaManan100 --- test/config.json | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/test/config.json b/test/config.json index 088d815ea7..ea76ab1b11 100644 --- a/test/config.json +++ b/test/config.json @@ -606,9 +606,36 @@ "RetryMax": 0, "Tags": [] }, - "vtgate_schematracker": { + "vtgate_schematracker_loadkeyspace": { "File": "unused.go", - "Args": ["vitess.io/vitess/go/test/endtoend/vtgate/schematracker"], + "Args": ["vitess.io/vitess/go/test/endtoend/vtgate/schematracker/loadkeyspace"], + "Command": [], + "Manual": false, + "Shard": "17", + "RetryMax": 0, + "Tags": [] + }, + "vtgate_schematracker_restarttablet": { + "File": "unused.go", + "Args": ["vitess.io/vitess/go/test/endtoend/vtgate/schematracker/restarttablet"], + "Command": [], + "Manual": false, + "Shard": "17", + "RetryMax": 0, + "Tags": [] + }, + "vtgate_schematracker_sharded": { + "File": "unused.go", + "Args": ["vitess.io/vitess/go/test/endtoend/vtgate/schematracker/sharded"], + "Command": [], + "Manual": false, + "Shard": "17", + "RetryMax": 0, + "Tags": [] + }, + "vtgate_schematracker_unsharded": { + "File": "unused.go", + "Args": ["vitess.io/vitess/go/test/endtoend/vtgate/schematracker/unsharded"], "Command": [], "Manual": false, "Shard": "17", From 15fd287bfb523d94ea30263c2397fcbc33b93c2e Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Thu, 10 Jun 2021 15:20:35 +0530 Subject: [PATCH 57/64] logged error in teardown Signed-off-by: GuptaManan100 --- go/test/endtoend/docker/vttestserver.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/go/test/endtoend/docker/vttestserver.go b/go/test/endtoend/docker/vttestserver.go index 994c55b155..e6bc93d4e6 100644 --- a/go/test/endtoend/docker/vttestserver.go +++ b/go/test/endtoend/docker/vttestserver.go @@ -25,6 +25,8 @@ import ( "strconv" "strings" "time" + + "vitess.io/vitess/go/vt/log" ) const ( @@ -52,7 +54,10 @@ func newVttestserver(dockerImage string, keyspaces []string, numShards []int, my func (v *vttestserver) teardown() { cmd := exec.Command("docker", "rm", "--force", "vttestserver-end2end-test") - _ = cmd.Run() + err := cmd.Run() + if err != nil { + log.Errorf("docker teardown failed :- %s", err.Error()) + } } // startDockerImage starts the docker image for the vttestserver From 0026d1c499c709c671abdacf27fe4a65b1fcd8eb Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Thu, 10 Jun 2021 16:10:57 +0530 Subject: [PATCH 58/64] move tests out of loaded shard 17 Signed-off-by: Harshit Gangal --- .../cluster_endtoend_vtgate_buffer.yml | 50 +++++++++++++++++++ .../cluster_endtoend_vtgate_concurrentdml.yml | 50 +++++++++++++++++++ ...cluster_endtoend_vtgate_readafterwrite.yml | 50 +++++++++++++++++++ .../cluster_endtoend_vtgate_reservedconn.yml | 50 +++++++++++++++++++ .../cluster_endtoend_vtgate_schema.yml | 50 +++++++++++++++++++ .../cluster_endtoend_vtgate_sequence.yml | 50 +++++++++++++++++++ .../cluster_endtoend_vtgate_setstatement.yml | 50 +++++++++++++++++++ .../cluster_endtoend_vtgate_topo.yml | 50 +++++++++++++++++++ .../cluster_endtoend_vtgate_transaction.yml | 50 +++++++++++++++++++ .../cluster_endtoend_vtgate_unsharded.yml | 50 +++++++++++++++++++ .../cluster_endtoend_vtgate_vschema.yml | 50 +++++++++++++++++++ .../cluster_endtoend_xb_recovery.yml | 50 +++++++++++++++++++ test/ci_workflow_gen.go | 12 +++++ test/config.json | 28 +++++------ 14 files changed, 626 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/cluster_endtoend_vtgate_buffer.yml create mode 100644 .github/workflows/cluster_endtoend_vtgate_concurrentdml.yml create mode 100644 .github/workflows/cluster_endtoend_vtgate_readafterwrite.yml create mode 100644 .github/workflows/cluster_endtoend_vtgate_reservedconn.yml create mode 100644 .github/workflows/cluster_endtoend_vtgate_schema.yml create mode 100644 .github/workflows/cluster_endtoend_vtgate_sequence.yml create mode 100644 .github/workflows/cluster_endtoend_vtgate_setstatement.yml create mode 100644 .github/workflows/cluster_endtoend_vtgate_topo.yml create mode 100644 .github/workflows/cluster_endtoend_vtgate_transaction.yml create mode 100644 .github/workflows/cluster_endtoend_vtgate_unsharded.yml create mode 100644 .github/workflows/cluster_endtoend_vtgate_vschema.yml create mode 100644 .github/workflows/cluster_endtoend_xb_recovery.yml diff --git a/.github/workflows/cluster_endtoend_vtgate_buffer.yml b/.github/workflows/cluster_endtoend_vtgate_buffer.yml new file mode 100644 index 0000000000..93c30528bd --- /dev/null +++ b/.github/workflows/cluster_endtoend_vtgate_buffer.yml @@ -0,0 +1,50 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (vtgate_buffer) +on: [push, pull_request] +jobs: + + build: + name: Run endtoend tests on Cluster (vtgate_buffer) + runs-on: ubuntu-18.04 + + steps: + - name: Set up Go + uses: actions/setup-go@v1 + with: + go-version: 1.15 + + - name: Tune the OS + run: | + echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + + # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 + - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file + run: | + echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts + # DON'T FORGET TO REMOVE CODE ABOVE WHEN ISSUE IS ADRESSED! + + - name: Check out code + uses: actions/checkout@v2 + + - name: Get dependencies + run: | + sudo apt-get update + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get install -y gnupg2 + sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get update + sudo apt-get install percona-xtrabackup-24 + + - name: Run cluster endtoend test + timeout-minutes: 30 + run: | + source build.env + eatmydata -- go run test.go -docker=false -print-log -follow -shard vtgate_buffer diff --git a/.github/workflows/cluster_endtoend_vtgate_concurrentdml.yml b/.github/workflows/cluster_endtoend_vtgate_concurrentdml.yml new file mode 100644 index 0000000000..4045c90ad7 --- /dev/null +++ b/.github/workflows/cluster_endtoend_vtgate_concurrentdml.yml @@ -0,0 +1,50 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (vtgate_concurrentdml) +on: [push, pull_request] +jobs: + + build: + name: Run endtoend tests on Cluster (vtgate_concurrentdml) + runs-on: ubuntu-18.04 + + steps: + - name: Set up Go + uses: actions/setup-go@v1 + with: + go-version: 1.15 + + - name: Tune the OS + run: | + echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + + # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 + - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file + run: | + echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts + # DON'T FORGET TO REMOVE CODE ABOVE WHEN ISSUE IS ADRESSED! + + - name: Check out code + uses: actions/checkout@v2 + + - name: Get dependencies + run: | + sudo apt-get update + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get install -y gnupg2 + sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get update + sudo apt-get install percona-xtrabackup-24 + + - name: Run cluster endtoend test + timeout-minutes: 30 + run: | + source build.env + eatmydata -- go run test.go -docker=false -print-log -follow -shard vtgate_concurrentdml diff --git a/.github/workflows/cluster_endtoend_vtgate_readafterwrite.yml b/.github/workflows/cluster_endtoend_vtgate_readafterwrite.yml new file mode 100644 index 0000000000..04102581bc --- /dev/null +++ b/.github/workflows/cluster_endtoend_vtgate_readafterwrite.yml @@ -0,0 +1,50 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (vtgate_readafterwrite) +on: [push, pull_request] +jobs: + + build: + name: Run endtoend tests on Cluster (vtgate_readafterwrite) + runs-on: ubuntu-18.04 + + steps: + - name: Set up Go + uses: actions/setup-go@v1 + with: + go-version: 1.15 + + - name: Tune the OS + run: | + echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + + # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 + - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file + run: | + echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts + # DON'T FORGET TO REMOVE CODE ABOVE WHEN ISSUE IS ADRESSED! + + - name: Check out code + uses: actions/checkout@v2 + + - name: Get dependencies + run: | + sudo apt-get update + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get install -y gnupg2 + sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get update + sudo apt-get install percona-xtrabackup-24 + + - name: Run cluster endtoend test + timeout-minutes: 30 + run: | + source build.env + eatmydata -- go run test.go -docker=false -print-log -follow -shard vtgate_readafterwrite diff --git a/.github/workflows/cluster_endtoend_vtgate_reservedconn.yml b/.github/workflows/cluster_endtoend_vtgate_reservedconn.yml new file mode 100644 index 0000000000..eaf68bd7df --- /dev/null +++ b/.github/workflows/cluster_endtoend_vtgate_reservedconn.yml @@ -0,0 +1,50 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (vtgate_reservedconn) +on: [push, pull_request] +jobs: + + build: + name: Run endtoend tests on Cluster (vtgate_reservedconn) + runs-on: ubuntu-18.04 + + steps: + - name: Set up Go + uses: actions/setup-go@v1 + with: + go-version: 1.15 + + - name: Tune the OS + run: | + echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + + # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 + - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file + run: | + echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts + # DON'T FORGET TO REMOVE CODE ABOVE WHEN ISSUE IS ADRESSED! + + - name: Check out code + uses: actions/checkout@v2 + + - name: Get dependencies + run: | + sudo apt-get update + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get install -y gnupg2 + sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get update + sudo apt-get install percona-xtrabackup-24 + + - name: Run cluster endtoend test + timeout-minutes: 30 + run: | + source build.env + eatmydata -- go run test.go -docker=false -print-log -follow -shard vtgate_reservedconn diff --git a/.github/workflows/cluster_endtoend_vtgate_schema.yml b/.github/workflows/cluster_endtoend_vtgate_schema.yml new file mode 100644 index 0000000000..0fdb3a3f0c --- /dev/null +++ b/.github/workflows/cluster_endtoend_vtgate_schema.yml @@ -0,0 +1,50 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (vtgate_schema) +on: [push, pull_request] +jobs: + + build: + name: Run endtoend tests on Cluster (vtgate_schema) + runs-on: ubuntu-18.04 + + steps: + - name: Set up Go + uses: actions/setup-go@v1 + with: + go-version: 1.15 + + - name: Tune the OS + run: | + echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + + # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 + - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file + run: | + echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts + # DON'T FORGET TO REMOVE CODE ABOVE WHEN ISSUE IS ADRESSED! + + - name: Check out code + uses: actions/checkout@v2 + + - name: Get dependencies + run: | + sudo apt-get update + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get install -y gnupg2 + sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get update + sudo apt-get install percona-xtrabackup-24 + + - name: Run cluster endtoend test + timeout-minutes: 30 + run: | + source build.env + eatmydata -- go run test.go -docker=false -print-log -follow -shard vtgate_schema diff --git a/.github/workflows/cluster_endtoend_vtgate_sequence.yml b/.github/workflows/cluster_endtoend_vtgate_sequence.yml new file mode 100644 index 0000000000..a8f4f55061 --- /dev/null +++ b/.github/workflows/cluster_endtoend_vtgate_sequence.yml @@ -0,0 +1,50 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (vtgate_sequence) +on: [push, pull_request] +jobs: + + build: + name: Run endtoend tests on Cluster (vtgate_sequence) + runs-on: ubuntu-18.04 + + steps: + - name: Set up Go + uses: actions/setup-go@v1 + with: + go-version: 1.15 + + - name: Tune the OS + run: | + echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + + # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 + - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file + run: | + echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts + # DON'T FORGET TO REMOVE CODE ABOVE WHEN ISSUE IS ADRESSED! + + - name: Check out code + uses: actions/checkout@v2 + + - name: Get dependencies + run: | + sudo apt-get update + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get install -y gnupg2 + sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get update + sudo apt-get install percona-xtrabackup-24 + + - name: Run cluster endtoend test + timeout-minutes: 30 + run: | + source build.env + eatmydata -- go run test.go -docker=false -print-log -follow -shard vtgate_sequence diff --git a/.github/workflows/cluster_endtoend_vtgate_setstatement.yml b/.github/workflows/cluster_endtoend_vtgate_setstatement.yml new file mode 100644 index 0000000000..2b168bdbbb --- /dev/null +++ b/.github/workflows/cluster_endtoend_vtgate_setstatement.yml @@ -0,0 +1,50 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (vtgate_setstatement) +on: [push, pull_request] +jobs: + + build: + name: Run endtoend tests on Cluster (vtgate_setstatement) + runs-on: ubuntu-18.04 + + steps: + - name: Set up Go + uses: actions/setup-go@v1 + with: + go-version: 1.15 + + - name: Tune the OS + run: | + echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + + # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 + - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file + run: | + echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts + # DON'T FORGET TO REMOVE CODE ABOVE WHEN ISSUE IS ADRESSED! + + - name: Check out code + uses: actions/checkout@v2 + + - name: Get dependencies + run: | + sudo apt-get update + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get install -y gnupg2 + sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get update + sudo apt-get install percona-xtrabackup-24 + + - name: Run cluster endtoend test + timeout-minutes: 30 + run: | + source build.env + eatmydata -- go run test.go -docker=false -print-log -follow -shard vtgate_setstatement diff --git a/.github/workflows/cluster_endtoend_vtgate_topo.yml b/.github/workflows/cluster_endtoend_vtgate_topo.yml new file mode 100644 index 0000000000..92b2c3407d --- /dev/null +++ b/.github/workflows/cluster_endtoend_vtgate_topo.yml @@ -0,0 +1,50 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (vtgate_topo) +on: [push, pull_request] +jobs: + + build: + name: Run endtoend tests on Cluster (vtgate_topo) + runs-on: ubuntu-18.04 + + steps: + - name: Set up Go + uses: actions/setup-go@v1 + with: + go-version: 1.15 + + - name: Tune the OS + run: | + echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + + # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 + - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file + run: | + echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts + # DON'T FORGET TO REMOVE CODE ABOVE WHEN ISSUE IS ADRESSED! + + - name: Check out code + uses: actions/checkout@v2 + + - name: Get dependencies + run: | + sudo apt-get update + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get install -y gnupg2 + sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get update + sudo apt-get install percona-xtrabackup-24 + + - name: Run cluster endtoend test + timeout-minutes: 30 + run: | + source build.env + eatmydata -- go run test.go -docker=false -print-log -follow -shard vtgate_topo diff --git a/.github/workflows/cluster_endtoend_vtgate_transaction.yml b/.github/workflows/cluster_endtoend_vtgate_transaction.yml new file mode 100644 index 0000000000..45d658266e --- /dev/null +++ b/.github/workflows/cluster_endtoend_vtgate_transaction.yml @@ -0,0 +1,50 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (vtgate_transaction) +on: [push, pull_request] +jobs: + + build: + name: Run endtoend tests on Cluster (vtgate_transaction) + runs-on: ubuntu-18.04 + + steps: + - name: Set up Go + uses: actions/setup-go@v1 + with: + go-version: 1.15 + + - name: Tune the OS + run: | + echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + + # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 + - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file + run: | + echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts + # DON'T FORGET TO REMOVE CODE ABOVE WHEN ISSUE IS ADRESSED! + + - name: Check out code + uses: actions/checkout@v2 + + - name: Get dependencies + run: | + sudo apt-get update + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get install -y gnupg2 + sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get update + sudo apt-get install percona-xtrabackup-24 + + - name: Run cluster endtoend test + timeout-minutes: 30 + run: | + source build.env + eatmydata -- go run test.go -docker=false -print-log -follow -shard vtgate_transaction diff --git a/.github/workflows/cluster_endtoend_vtgate_unsharded.yml b/.github/workflows/cluster_endtoend_vtgate_unsharded.yml new file mode 100644 index 0000000000..18f69aae1d --- /dev/null +++ b/.github/workflows/cluster_endtoend_vtgate_unsharded.yml @@ -0,0 +1,50 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (vtgate_unsharded) +on: [push, pull_request] +jobs: + + build: + name: Run endtoend tests on Cluster (vtgate_unsharded) + runs-on: ubuntu-18.04 + + steps: + - name: Set up Go + uses: actions/setup-go@v1 + with: + go-version: 1.15 + + - name: Tune the OS + run: | + echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + + # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 + - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file + run: | + echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts + # DON'T FORGET TO REMOVE CODE ABOVE WHEN ISSUE IS ADRESSED! + + - name: Check out code + uses: actions/checkout@v2 + + - name: Get dependencies + run: | + sudo apt-get update + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get install -y gnupg2 + sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get update + sudo apt-get install percona-xtrabackup-24 + + - name: Run cluster endtoend test + timeout-minutes: 30 + run: | + source build.env + eatmydata -- go run test.go -docker=false -print-log -follow -shard vtgate_unsharded diff --git a/.github/workflows/cluster_endtoend_vtgate_vschema.yml b/.github/workflows/cluster_endtoend_vtgate_vschema.yml new file mode 100644 index 0000000000..aa8c3ed1c7 --- /dev/null +++ b/.github/workflows/cluster_endtoend_vtgate_vschema.yml @@ -0,0 +1,50 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (vtgate_vschema) +on: [push, pull_request] +jobs: + + build: + name: Run endtoend tests on Cluster (vtgate_vschema) + runs-on: ubuntu-18.04 + + steps: + - name: Set up Go + uses: actions/setup-go@v1 + with: + go-version: 1.15 + + - name: Tune the OS + run: | + echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + + # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 + - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file + run: | + echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts + # DON'T FORGET TO REMOVE CODE ABOVE WHEN ISSUE IS ADRESSED! + + - name: Check out code + uses: actions/checkout@v2 + + - name: Get dependencies + run: | + sudo apt-get update + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get install -y gnupg2 + sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get update + sudo apt-get install percona-xtrabackup-24 + + - name: Run cluster endtoend test + timeout-minutes: 30 + run: | + source build.env + eatmydata -- go run test.go -docker=false -print-log -follow -shard vtgate_vschema diff --git a/.github/workflows/cluster_endtoend_xb_recovery.yml b/.github/workflows/cluster_endtoend_xb_recovery.yml new file mode 100644 index 0000000000..ec15f6b9b8 --- /dev/null +++ b/.github/workflows/cluster_endtoend_xb_recovery.yml @@ -0,0 +1,50 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (xb_recovery) +on: [push, pull_request] +jobs: + + build: + name: Run endtoend tests on Cluster (xb_recovery) + runs-on: ubuntu-18.04 + + steps: + - name: Set up Go + uses: actions/setup-go@v1 + with: + go-version: 1.15 + + - name: Tune the OS + run: | + echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + + # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 + - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file + run: | + echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts + # DON'T FORGET TO REMOVE CODE ABOVE WHEN ISSUE IS ADRESSED! + + - name: Check out code + uses: actions/checkout@v2 + + - name: Get dependencies + run: | + sudo apt-get update + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get install -y gnupg2 + sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get update + sudo apt-get install percona-xtrabackup-24 + + - name: Run cluster endtoend test + timeout-minutes: 30 + run: | + source build.env + eatmydata -- go run test.go -docker=false -print-log -follow -shard xb_recovery diff --git a/test/ci_workflow_gen.go b/test/ci_workflow_gen.go index a1279a073b..26d971b4ed 100644 --- a/test/ci_workflow_gen.go +++ b/test/ci_workflow_gen.go @@ -67,6 +67,18 @@ var ( "tabletmanager_throttler_custom_config", "tabletmanager_tablegc", "vtorc", + "vtgate_buffer", + "vtgate_concurrentdml", + "vtgate_schema", + "vtgate_sequence", + "vtgate_setstatement", + "vtgate_reservedconn", + "vtgate_transaction", + "vtgate_unsharded", + "vtgate_vschema", + "vtgate_readafterwrite", + "vtgate_topo", + "xb_recovery", } // TODO: currently some percona tools including xtrabackup are installed on all clusters, we can possibly optimize // this by only installing them in the required clusters diff --git a/test/config.json b/test/config.json index 1cae4d07ff..da49acc220 100644 --- a/test/config.json +++ b/test/config.json @@ -584,7 +584,7 @@ "Args": ["vitess.io/vitess/go/test/endtoend/vtgate/buffer"], "Command": [], "Manual": false, - "Shard": "17", + "Shard": "vtgate_buffer", "RetryMax": 0, "Tags": [] }, @@ -593,7 +593,7 @@ "Args": ["vitess.io/vitess/go/test/endtoend/vtgate/concurrentdml"], "Command": [], "Manual": false, - "Shard": "17", + "Shard": "vtgate_concurrentdml", "RetryMax": 0, "Tags": [] }, @@ -602,7 +602,7 @@ "Args": ["vitess.io/vitess/go/test/endtoend/vtgate/schema"], "Command": [], "Manual": false, - "Shard": "17", + "Shard": "vtgate_schema", "RetryMax": 0, "Tags": [] }, @@ -611,7 +611,7 @@ "Args": ["vitess.io/vitess/go/test/endtoend/vtgate/sequence"], "Command": [], "Manual": false, - "Shard": "17", + "Shard": "vtgate_sequence", "RetryMax": 0, "Tags": [] }, @@ -620,7 +620,7 @@ "Args": ["vitess.io/vitess/go/test/endtoend/vtgate/reservedconn"], "Command": [], "Manual": false, - "Shard": "17", + "Shard": "vtgate_setstatement", "RetryMax": 0, "Tags": [] }, @@ -629,7 +629,7 @@ "Args": ["vitess.io/vitess/go/test/endtoend/vtgate/reservedconn/reconnect1"], "Command": [], "Manual": false, - "Shard": "17", + "Shard": "vtgate_reservedconn", "RetryMax": 0, "Tags": [] }, @@ -638,7 +638,7 @@ "Args": ["vitess.io/vitess/go/test/endtoend/vtgate/reservedconn/reconnect2"], "Command": [], "Manual": false, - "Shard": "17", + "Shard": "vtgate_reservedconn", "RetryMax": 0, "Tags": [] }, @@ -647,7 +647,7 @@ "Args": ["vitess.io/vitess/go/test/endtoend/vtgate/transaction"], "Command": [], "Manual": false, - "Shard": "17", + "Shard": "vtgate_transaction", "RetryMax": 0, "Tags": [] }, @@ -656,7 +656,7 @@ "Args": ["vitess.io/vitess/go/test/endtoend/vtgate/unsharded"], "Command": [], "Manual": false, - "Shard": "17", + "Shard": "vtgate_unsharded", "RetryMax": 0, "Tags": [] }, @@ -665,7 +665,7 @@ "Args": ["vitess.io/vitess/go/test/endtoend/vtgate/vschema"], "Command": [], "Manual": false, - "Shard": "17", + "Shard": "vtgate_vschema", "RetryMax": 0, "Tags": [] }, @@ -674,7 +674,7 @@ "Args": ["vitess.io/vitess/go/test/endtoend/vtgate/readafterwrite"], "Command": [], "Manual": false, - "Shard": "17", + "Shard": "vtgate_readafterwrite", "RetryMax": 0, "Tags": [] }, @@ -701,7 +701,7 @@ "Args": ["vitess.io/vitess/go/test/endtoend/topotest/etcd2"], "Command": [], "Manual": false, - "Shard": "17", + "Shard": "vtgate_topo", "RetryMax": 0, "Tags": [] }, @@ -710,7 +710,7 @@ "Args": ["vitess.io/vitess/go/test/endtoend/vtgate/errors_as_warnings"], "Command": [], "Manual": false, - "Shard": "17", + "Shard": "vtgate_topo", "RetryMax": 0, "Tags": [] }, @@ -737,7 +737,7 @@ "Args": ["vitess.io/vitess/go/test/endtoend/recovery/xtrabackup"], "Command": [], "Manual": false, - "Shard": "17", + "Shard": "xb_recovery", "RetryMax": 0, "Tags": [] }, From 19457309f6929e0e03f1db8cd15d279d9aa077cb Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Thu, 10 Jun 2021 15:16:10 +0200 Subject: [PATCH 59/64] Add VReplication lag metric in the form of a gauge for easier display in VTAdmin Signed-off-by: Rohit Nayak --- go/vt/binlog/binlogplayer/binlog_player.go | 5 +++++ .../vttablet/tabletmanager/vreplication/stats.go | 15 +++++++++++++++ .../tabletmanager/vreplication/vplayer.go | 4 ++++ 3 files changed, 24 insertions(+) diff --git a/go/vt/binlog/binlogplayer/binlog_player.go b/go/vt/binlog/binlogplayer/binlog_player.go index 5683f5dd2c..313fd8ef83 100644 --- a/go/vt/binlog/binlogplayer/binlog_player.go +++ b/go/vt/binlog/binlogplayer/binlog_player.go @@ -98,6 +98,9 @@ type Stats struct { CopyLoopCount *stats.Counter ErrorCounts *stats.CountersWithMultiLabels NoopQueryCount *stats.CountersWithSingleLabel + + VReplicationLags *stats.Timings + VReplicationLagRates *stats.Rates } // RecordHeartbeat updates the time the last heartbeat from vstreamer was seen @@ -154,6 +157,8 @@ func NewStats() *Stats { bps.CopyLoopCount = stats.NewCounter("", "") bps.ErrorCounts = stats.NewCountersWithMultiLabels("", "", []string{"type"}) bps.NoopQueryCount = stats.NewCountersWithSingleLabel("", "", "Statement", "") + bps.VReplicationLags = stats.NewTimings("", "", "") + bps.VReplicationLagRates = stats.NewRates("", bps.VReplicationLags, 15*60/5, 5*time.Second) return bps } diff --git a/go/vt/vttablet/tabletmanager/vreplication/stats.go b/go/vt/vttablet/tabletmanager/vreplication/stats.go index b0a571ebb5..4cb4a9fe3c 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/stats.go +++ b/go/vt/vttablet/tabletmanager/vreplication/stats.go @@ -113,6 +113,21 @@ func (st *vrStats) register() { return result }) + stats.NewRateFunc( + "VReplicationLag", + "vreplication lag per stream", + func() map[string][]float64 { + st.mu.Lock() + defer st.mu.Unlock() + result := make(map[string][]float64) + for _, ct := range st.controllers { + for k, v := range ct.blpStats.VReplicationLagRates.Get() { + result[k] = v + } + } + return result + }) + stats.Publish("VReplicationSource", stats.StringMapFunc(func() map[string]string { st.mu.Lock() defer st.mu.Unlock() diff --git a/go/vt/vttablet/tabletmanager/vreplication/vplayer.go b/go/vt/vttablet/tabletmanager/vreplication/vplayer.go index 9933d0db7d..f4a273f1e0 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vplayer.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vplayer.go @@ -21,6 +21,7 @@ import ( "fmt" "io" "math" + "strconv" "strings" "time" @@ -331,6 +332,7 @@ func (vp *vplayer) applyEvents(ctx context.Context, relay *relayLog) error { // TODO(sougou): if we also stored the time of the last event, we // can estimate this value more accurately. defer vp.vr.stats.SecondsBehindMaster.Set(math.MaxInt64) + defer vp.vr.stats.VReplicationLags.Add(strconv.Itoa(int(vp.vr.id)), math.MaxInt64) var sbm int64 = -1 for { // check throttler. @@ -347,6 +349,7 @@ func (vp *vplayer) applyEvents(ctx context.Context, relay *relayLog) error { if len(items) == 0 { behind := time.Now().UnixNano() - vp.lastTimestampNs - vp.timeOffsetNs vp.vr.stats.SecondsBehindMaster.Set(behind / 1e9) + vp.vr.stats.VReplicationLags.Add(strconv.Itoa(int(vp.vr.id)), time.Duration(behind/1e9)*time.Second) } // Empty transactions are saved at most once every idleTimeout. // This covers two situations: @@ -401,6 +404,7 @@ func (vp *vplayer) applyEvents(ctx context.Context, relay *relayLog) error { } if sbm >= 0 { vp.vr.stats.SecondsBehindMaster.Set(sbm) + vp.vr.stats.VReplicationLags.Add(strconv.Itoa(int(vp.vr.id)), time.Duration(sbm)*time.Second) } } From 172d139ed7152dd7200d92c6c458dbc33517e4e9 Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Thu, 10 Jun 2021 19:54:39 +0530 Subject: [PATCH 60/64] split shard 15 tests Signed-off-by: Harshit Gangal --- .../workflows/cluster_endtoend_resharding.yml | 50 +++++++++++++++++++ .../cluster_endtoend_resharding_bytes.yml | 50 +++++++++++++++++++ test/ci_workflow_gen.go | 2 + test/config.json | 4 +- 4 files changed, 104 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/cluster_endtoend_resharding.yml create mode 100644 .github/workflows/cluster_endtoend_resharding_bytes.yml diff --git a/.github/workflows/cluster_endtoend_resharding.yml b/.github/workflows/cluster_endtoend_resharding.yml new file mode 100644 index 0000000000..6a724008a1 --- /dev/null +++ b/.github/workflows/cluster_endtoend_resharding.yml @@ -0,0 +1,50 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (resharding) +on: [push, pull_request] +jobs: + + build: + name: Run endtoend tests on Cluster (resharding) + runs-on: ubuntu-18.04 + + steps: + - name: Set up Go + uses: actions/setup-go@v1 + with: + go-version: 1.15 + + - name: Tune the OS + run: | + echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + + # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 + - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file + run: | + echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts + # DON'T FORGET TO REMOVE CODE ABOVE WHEN ISSUE IS ADRESSED! + + - name: Check out code + uses: actions/checkout@v2 + + - name: Get dependencies + run: | + sudo apt-get update + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get install -y gnupg2 + sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get update + sudo apt-get install percona-xtrabackup-24 + + - name: Run cluster endtoend test + timeout-minutes: 30 + run: | + source build.env + eatmydata -- go run test.go -docker=false -print-log -follow -shard resharding diff --git a/.github/workflows/cluster_endtoend_resharding_bytes.yml b/.github/workflows/cluster_endtoend_resharding_bytes.yml new file mode 100644 index 0000000000..11d45b0853 --- /dev/null +++ b/.github/workflows/cluster_endtoend_resharding_bytes.yml @@ -0,0 +1,50 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (resharding_bytes) +on: [push, pull_request] +jobs: + + build: + name: Run endtoend tests on Cluster (resharding_bytes) + runs-on: ubuntu-18.04 + + steps: + - name: Set up Go + uses: actions/setup-go@v1 + with: + go-version: 1.15 + + - name: Tune the OS + run: | + echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + + # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 + - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file + run: | + echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts + # DON'T FORGET TO REMOVE CODE ABOVE WHEN ISSUE IS ADRESSED! + + - name: Check out code + uses: actions/checkout@v2 + + - name: Get dependencies + run: | + sudo apt-get update + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get install -y gnupg2 + sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get update + sudo apt-get install percona-xtrabackup-24 + + - name: Run cluster endtoend test + timeout-minutes: 30 + run: | + source build.env + eatmydata -- go run test.go -docker=false -print-log -follow -shard resharding_bytes diff --git a/test/ci_workflow_gen.go b/test/ci_workflow_gen.go index 26d971b4ed..83797ab5fe 100644 --- a/test/ci_workflow_gen.go +++ b/test/ci_workflow_gen.go @@ -79,6 +79,8 @@ var ( "vtgate_readafterwrite", "vtgate_topo", "xb_recovery", + "resharding", + "resharding_bytes", } // TODO: currently some percona tools including xtrabackup are installed on all clusters, we can possibly optimize // this by only installing them in the required clusters diff --git a/test/config.json b/test/config.json index da49acc220..cf98ec5ef8 100644 --- a/test/config.json +++ b/test/config.json @@ -418,7 +418,7 @@ "Args": ["vitess.io/vitess/go/test/endtoend/sharding/resharding/v3"], "Command": [], "Manual": false, - "Shard": "15", + "Shard": "resharding", "RetryMax": 0, "Tags": [ "worker_test" @@ -429,7 +429,7 @@ "Args": ["vitess.io/vitess/go/test/endtoend/sharding/resharding/string"], "Command": [], "Manual": false, - "Shard": "15", + "Shard": "resharding_bytes", "RetryMax": 0, "Tags": [ "worker_test" From 639106cab999b6aeb98997703df44affc8dc92bb Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Thu, 10 Jun 2021 20:14:59 +0530 Subject: [PATCH 61/64] added missing tests to CI Signed-off-by: Harshit Gangal --- ...e.yml => cluster_endtoend_vtgate_gen4.yml} | 6 +- ...yml => cluster_endtoend_vtgate_vindex.yml} | 6 +- test/ci_workflow_gen.go | 10 ++-- test/config.json | 60 ++++++++++++++++++- 4 files changed, 68 insertions(+), 14 deletions(-) rename .github/workflows/{cluster_endtoend_vtgate_sequence.yml => cluster_endtoend_vtgate_gen4.yml} (93%) rename .github/workflows/{cluster_endtoend_vtgate_setstatement.yml => cluster_endtoend_vtgate_vindex.yml} (92%) diff --git a/.github/workflows/cluster_endtoend_vtgate_sequence.yml b/.github/workflows/cluster_endtoend_vtgate_gen4.yml similarity index 93% rename from .github/workflows/cluster_endtoend_vtgate_sequence.yml rename to .github/workflows/cluster_endtoend_vtgate_gen4.yml index a8f4f55061..6afb9f5676 100644 --- a/.github/workflows/cluster_endtoend_vtgate_sequence.yml +++ b/.github/workflows/cluster_endtoend_vtgate_gen4.yml @@ -1,11 +1,11 @@ # DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" -name: Cluster (vtgate_sequence) +name: Cluster (vtgate_gen4) on: [push, pull_request] jobs: build: - name: Run endtoend tests on Cluster (vtgate_sequence) + name: Run endtoend tests on Cluster (vtgate_gen4) runs-on: ubuntu-18.04 steps: @@ -47,4 +47,4 @@ jobs: timeout-minutes: 30 run: | source build.env - eatmydata -- go run test.go -docker=false -print-log -follow -shard vtgate_sequence + eatmydata -- go run test.go -docker=false -print-log -follow -shard vtgate_gen4 diff --git a/.github/workflows/cluster_endtoend_vtgate_setstatement.yml b/.github/workflows/cluster_endtoend_vtgate_vindex.yml similarity index 92% rename from .github/workflows/cluster_endtoend_vtgate_setstatement.yml rename to .github/workflows/cluster_endtoend_vtgate_vindex.yml index 2b168bdbbb..e157c39db7 100644 --- a/.github/workflows/cluster_endtoend_vtgate_setstatement.yml +++ b/.github/workflows/cluster_endtoend_vtgate_vindex.yml @@ -1,11 +1,11 @@ # DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" -name: Cluster (vtgate_setstatement) +name: Cluster (vtgate_vindex) on: [push, pull_request] jobs: build: - name: Run endtoend tests on Cluster (vtgate_setstatement) + name: Run endtoend tests on Cluster (vtgate_vindex) runs-on: ubuntu-18.04 steps: @@ -47,4 +47,4 @@ jobs: timeout-minutes: 30 run: | source build.env - eatmydata -- go run test.go -docker=false -print-log -follow -shard vtgate_setstatement + eatmydata -- go run test.go -docker=false -print-log -follow -shard vtgate_vindex diff --git a/test/ci_workflow_gen.go b/test/ci_workflow_gen.go index 83797ab5fe..dc79e4b043 100644 --- a/test/ci_workflow_gen.go +++ b/test/ci_workflow_gen.go @@ -69,15 +69,15 @@ var ( "vtorc", "vtgate_buffer", "vtgate_concurrentdml", - "vtgate_schema", - "vtgate_sequence", - "vtgate_setstatement", + "vtgate_gen4", + "vtgate_readafterwrite", "vtgate_reservedconn", + "vtgate_schema", + "vtgate_topo", "vtgate_transaction", "vtgate_unsharded", + "vtgate_vindex", "vtgate_vschema", - "vtgate_readafterwrite", - "vtgate_topo", "xb_recovery", "resharding", "resharding_bytes", diff --git a/test/config.json b/test/config.json index cf98ec5ef8..f9f099fa51 100644 --- a/test/config.json +++ b/test/config.json @@ -611,7 +611,7 @@ "Args": ["vitess.io/vitess/go/test/endtoend/vtgate/sequence"], "Command": [], "Manual": false, - "Shard": "vtgate_sequence", + "Shard": "17", "RetryMax": 0, "Tags": [] }, @@ -620,7 +620,7 @@ "Args": ["vitess.io/vitess/go/test/endtoend/vtgate/reservedconn"], "Command": [], "Manual": false, - "Shard": "vtgate_setstatement", + "Shard": "vtgate_reservedconn", "RetryMax": 0, "Tags": [] }, @@ -651,6 +651,15 @@ "RetryMax": 0, "Tags": [] }, + "vtgate_transaction_rollback": { + "File": "unused.go", + "Args": ["vitess.io/vitess/go/test/endtoend/vtgate/transaction/rollback"], + "Command": [], + "Manual": false, + "Shard": "vtgate_transaction", + "RetryMax": 0, + "Tags": [] + }, "vtgate_unsharded": { "File": "unused.go", "Args": ["vitess.io/vitess/go/test/endtoend/vtgate/unsharded"], @@ -678,6 +687,33 @@ "RetryMax": 0, "Tags": [] }, + "vtgate_dbddlplugin": { + "File": "unused.go", + "Args": ["vitess.io/vitess/go/test/endtoend/vtgate/createdb_plugin"], + "Command": [], + "Manual": false, + "Shard": "17", + "RetryMax": 0, + "Tags": [] + }, + "vtgate_gen4": { + "File": "unused.go", + "Args": ["vitess.io/vitess/go/test/endtoend/vtgate/gen4"], + "Command": [], + "Manual": false, + "Shard": "vtgate_gen4", + "RetryMax": 0, + "Tags": [] + }, + "vtgate_watchkeyspace": { + "File": "unused.go", + "Args": ["vitess.io/vitess/go/test/endtoend/vtgate/keyspace_watches"], + "Command": [], + "Manual": false, + "Shard": "vtgate_topo", + "RetryMax": 0, + "Tags": [] + }, "topo_zk2": { "File": "unused.go", "Args": ["vitess.io/vitess/go/test/endtoend/topotest/zk2", "--topo-flavor=zk2"], @@ -710,7 +746,25 @@ "Args": ["vitess.io/vitess/go/test/endtoend/vtgate/errors_as_warnings"], "Command": [], "Manual": false, - "Shard": "vtgate_topo", + "Shard": "17", + "RetryMax": 0, + "Tags": [] + }, + "prefixfanout": { + "File": "unused.go", + "Args": ["vitess.io/vitess/go/test/endtoend/vtgate/prefixfanout"], + "Command": [], + "Manual": false, + "Shard": "vtgate_vindex", + "RetryMax": 0, + "Tags": [] + }, + "vindex_bindvars": { + "File": "unused.go", + "Args": ["vitess.io/vitess/go/test/endtoend/vtgate/vindex_bindvars"], + "Command": [], + "Manual": false, + "Shard": "vtgate_vindex", "RetryMax": 0, "Tags": [] }, From ba8f8c4e080c58469ea26d268903feae76af5979 Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Thu, 10 Jun 2021 21:04:59 +0530 Subject: [PATCH 62/64] clear data in tables after test Signed-off-by: Harshit Gangal --- .../vtgate/transaction/rollback/txn_rollback_shutdown_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/test/endtoend/vtgate/transaction/rollback/txn_rollback_shutdown_test.go b/go/test/endtoend/vtgate/transaction/rollback/txn_rollback_shutdown_test.go index 04ee776006..9b4640a827 100644 --- a/go/test/endtoend/vtgate/transaction/rollback/txn_rollback_shutdown_test.go +++ b/go/test/endtoend/vtgate/transaction/rollback/txn_rollback_shutdown_test.go @@ -160,5 +160,5 @@ func TestErrorInAutocommitSession(t *testing.T) { // if we have properly working autocommit code, both the successful inserts should be visible to a second // connection, even if we have not done an explicit commit - assert.Equal(t, `[[INT64(1) VARCHAR("foo")] [INT64(2) VARCHAR("baz")]]`, fmt.Sprintf("%v", result.Rows)) + assert.Equal(t, `[[INT64(1) VARCHAR("foo")] [INT64(2) VARCHAR("baz")] [INT64(3) VARCHAR("mark")] [INT64(4) VARCHAR("doug")]]`, fmt.Sprintf("%v", result.Rows)) } From 2034f8973e4ba1eb848a5cf518a9b8d52a83e59a Mon Sep 17 00:00:00 2001 From: Sara Bee <855595+doeg@users.noreply.github.com> Date: Fri, 11 Jun 2021 06:20:12 -0400 Subject: [PATCH 63/64] [vtadmin-web] Small bugfix where stream source displayed instead of target Signed-off-by: Sara Bee <855595+doeg@users.noreply.github.com> --- web/vtadmin/src/components/routes/workflow/WorkflowStreams.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/vtadmin/src/components/routes/workflow/WorkflowStreams.tsx b/web/vtadmin/src/components/routes/workflow/WorkflowStreams.tsx index 81272cb7d4..b9c8a472a7 100644 --- a/web/vtadmin/src/components/routes/workflow/WorkflowStreams.tsx +++ b/web/vtadmin/src/components/routes/workflow/WorkflowStreams.tsx @@ -89,7 +89,7 @@ export const WorkflowStreams = ({ clusterID, keyspace, name }: Props) => { {target ? ( - {source} + {target} ) : ( N/A From e36e4b75144b570dd08532b579b9780ad688c5e4 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Sat, 12 Jun 2021 11:43:00 -0400 Subject: [PATCH 64/64] [vtctldserver] Update tests to prevent races during tmclient init Signed-off-by: Andrew Mason --- go/vt/vtctl/grpcvtctldserver/server_test.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/go/vt/vtctl/grpcvtctldserver/server_test.go b/go/vt/vtctl/grpcvtctldserver/server_test.go index 3fb15bf0f1..3db13543e1 100644 --- a/go/vt/vtctl/grpcvtctldserver/server_test.go +++ b/go/vt/vtctl/grpcvtctldserver/server_test.go @@ -1328,7 +1328,9 @@ func TestDeleteCellsAlias(t *testing.T) { require.NoError(t, err, "test setup failed") } - vtctld := NewVtctldServer(tt.ts) + vtctld := testutil.NewVtctldServerWithTabletManagerClient(t, tt.ts, nil, func(ts *topo.Server) vtctlservicepb.VtctldServer { + return NewVtctldServer(ts) + }) _, err := vtctld.DeleteCellsAlias(ctx, tt.req) if tt.shouldErr { assert.Error(t, err) @@ -2719,6 +2721,8 @@ func TestFindAllShardsInKeyspace(t *testing.T) { } func TestGetBackups(t *testing.T) { + t.Parallel() + ctx := context.Background() ts := memorytopo.NewServer() vtctld := testutil.NewVtctldServerWithTabletManagerClient(t, ts, nil, func(ts *topo.Server) vtctlservicepb.VtctldServer { @@ -3003,7 +3007,9 @@ func TestGetRoutingRules(t *testing.T) { factory.SetError(errors.New("topo down for testing")) } - vtctld := NewVtctldServer(ts) + vtctld := testutil.NewVtctldServerWithTabletManagerClient(t, ts, nil, func(ts *topo.Server) vtctlservicepb.VtctldServer { + return NewVtctldServer(ts) + }) resp, err := vtctld.GetRoutingRules(ctx, &vtctldatapb.GetRoutingRulesRequest{}) if tt.shouldErr { assert.Error(t, err) @@ -4515,7 +4521,9 @@ func TestRebuildVSchemaGraph(t *testing.T) { factory.SetError(errors.New("topo down for testing")) } - vtctld := NewVtctldServer(ts) + vtctld := testutil.NewVtctldServerWithTabletManagerClient(t, ts, nil, func(ts *topo.Server) vtctlservicepb.VtctldServer { + return NewVtctldServer(ts) + }) _, err := vtctld.RebuildVSchemaGraph(ctx, req) if tt.shouldErr { assert.Error(t, err)