From d570b99a14503fb8882048c3bf8781104d26a07c Mon Sep 17 00:00:00 2001 From: Michael Demmer Date: Mon, 26 Aug 2019 20:49:14 -0700 Subject: [PATCH 1/3] support more expression types in vtexplain Rework the vtexplain simulated mysql to handle more expression types when calculating fake column results based on inferences off the select expression. Also change the engine so that it logs an error instead of failing outright, since the column values aren't always needed for it to effectively simulate a query. Signed-off-by: Michael Demmer --- .../multi-output/selectsharded-output.txt | 35 ++++++++ .../testdata/selectsharded-queries.sql | 12 ++- go/vt/vtexplain/vtexplain_vttablet.go | 79 +++++++++++-------- 3 files changed, 91 insertions(+), 35 deletions(-) diff --git a/go/vt/vtexplain/testdata/multi-output/selectsharded-output.txt b/go/vt/vtexplain/testdata/multi-output/selectsharded-output.txt index 6f458bdd8b..198307bed3 100644 --- a/go/vt/vtexplain/testdata/multi-output/selectsharded-output.txt +++ b/go/vt/vtexplain/testdata/multi-output/selectsharded-output.txt @@ -126,3 +126,38 @@ select * from name_info order by info /* select * and order by varchar column */ 1 ks_sharded/c0-: select name, info, weight_string(info) from name_info order by info asc limit 10001 /* select * and order by varchar column */ ---------------------------------------------------------------------- +select distinct(name) from user where id = 1 /* select distinct */ + +1 ks_sharded/-40: select distinct (name) from user where id = 1 limit 10001 /* select distinct */ + +---------------------------------------------------------------------- +select distinct name from user where id = 1 /* select distinct */ + +1 ks_sharded/-40: select distinct name from user where id = 1 limit 10001 /* select distinct */ + +---------------------------------------------------------------------- +select id, substring(name, 1, -1) from user where id = 123 /* select substring */ + +1 ks_sharded/-40: select id, substr(name, 1, -1) from user where id = 123 limit 10001 /* select substring */ + +---------------------------------------------------------------------- +select id, substring_index(name, '123456', -1) from user where id = 123 /* select substring_index */ + +1 ks_sharded/-40: select id, substring_index(name, '123456', -1) from user where id = 123 limit 10001 /* select substring_index */ + +---------------------------------------------------------------------- +select id, case when name = 'alice' then 'ALICE' when name = 'bob' then 'BOB' end as name from user where id = 1 /* select case */ + +1 ks_sharded/-40: select id, case when name = 'alice' then 'ALICE' when name = 'bob' then 'BOB' end as name from user where id = 1 limit 10001 /* select case */ + +---------------------------------------------------------------------- +select id, case when name = 'alice' then 'ALICE' when name = 'bob' then 'BOB' else 'OTHER' end as name from user where id = 1 /* select case */ + +1 ks_sharded/-40: select id, case when name = 'alice' then 'ALICE' when name = 'bob' then 'BOB' else 'OTHER' end as name from user where id = 1 limit 10001 /* select case */ + +---------------------------------------------------------------------- +select id, case when substr(name, 1, 5) = 'alice' then 'ALICE' when name = 'bob' then 'BOB' else 'OTHER' end as name from user where id = 1 /* select case */ + +1 ks_sharded/-40: select id, case when substr(name, 1, 5) = 'alice' then 'ALICE' when name = 'bob' then 'BOB' else 'OTHER' end as name from user where id = 1 limit 10001 /* select case */ + +---------------------------------------------------------------------- diff --git a/go/vt/vtexplain/testdata/selectsharded-queries.sql b/go/vt/vtexplain/testdata/selectsharded-queries.sql index a51570ca83..6c3f7476f0 100644 --- a/go/vt/vtexplain/testdata/selectsharded-queries.sql +++ b/go/vt/vtexplain/testdata/selectsharded-queries.sql @@ -19,4 +19,14 @@ select name from user where id in (select id from t1) /* non-correlated subquery select name from user where id not in (select id from t1) /* non-correlated subquery in NOT IN clause */; select name from user where exists (select id from t1) /* non-correlated subquery as EXISTS */; -select * from name_info order by info /* select * and order by varchar column */ +select * from name_info order by info /* select * and order by varchar column */; + +select distinct(name) from user where id = 1 /* select distinct */; +select distinct name from user where id = 1 /* select distinct */; + +select id, substring(name, 1, -1) from user where id = 123 /* select substring */; +select id, substring_index(name, '123456', -1) from user where id = 123 /* select substring_index */; + +select id, case when name = 'alice' then 'ALICE' when name = 'bob' then 'BOB' end as name from user where id = 1 /* select case */; +select id, case when name = 'alice' then 'ALICE' when name = 'bob' then 'BOB' else 'OTHER' end as name from user where id = 1 /* select case */; +select id, case when substr(name, 1, 5) = 'alice' then 'ALICE' when name = 'bob' then 'BOB' else 'OTHER' end as name from user where id = 1 /* select case */; diff --git a/go/vt/vtexplain/vtexplain_vttablet.go b/go/vt/vtexplain/vtexplain_vttablet.go index 7c00864f2c..e45a822d37 100644 --- a/go/vt/vtexplain/vtexplain_vttablet.go +++ b/go/vt/vtexplain/vtexplain_vttablet.go @@ -19,6 +19,7 @@ package vtexplain import ( "encoding/json" "fmt" + "reflect" "strings" "sync" @@ -531,40 +532,7 @@ func (t *explainTablet) HandleQuery(c *mysql.Conn, query string, callback func(* for _, node := range selStmt.SelectExprs { switch node := node.(type) { case *sqlparser.AliasedExpr: - switch node := node.Expr.(type) { - case *sqlparser.ColName: - col := strings.ToLower(node.Name.String()) - colType := colTypeMap[col] - if colType == querypb.Type_NULL_TYPE { - return fmt.Errorf("invalid column %s", col) - } - colNames = append(colNames, col) - colTypes = append(colTypes, colType) - case *sqlparser.FuncExpr: - // As a shortcut, functions are integral types - colNames = append(colNames, sqlparser.String(node)) - colTypes = append(colTypes, querypb.Type_INT32) - case *sqlparser.SQLVal: - colNames = append(colNames, sqlparser.String(node)) - switch node.Type { - case sqlparser.IntVal: - fallthrough - case sqlparser.HexNum: - fallthrough - case sqlparser.HexVal: - fallthrough - case sqlparser.BitVal: - colTypes = append(colTypes, querypb.Type_INT32) - case sqlparser.StrVal: - colTypes = append(colTypes, querypb.Type_VARCHAR) - case sqlparser.FloatVal: - colTypes = append(colTypes, querypb.Type_FLOAT64) - default: - return fmt.Errorf("unsupported sql value %s", sqlparser.String(node)) - } - default: - return fmt.Errorf("unsupported select expression %s", sqlparser.String(node)) - } + colNames, colTypes = inferColTypeFromExpr(node.Expr, colTypeMap, colNames, colTypes) case *sqlparser.StarExpr: for col, colType := range colTypeMap { colNames = append(colNames, col) @@ -615,3 +583,46 @@ func (t *explainTablet) HandleQuery(c *mysql.Conn, query string, callback func(* return callback(result) } + +func inferColTypeFromExpr(node sqlparser.Expr, colTypeMap map[string]querypb.Type, colNames []string, colTypes []querypb.Type) ([]string, []querypb.Type) { + switch node := node.(type) { + case *sqlparser.ColName: + col := strings.ToLower(node.Name.String()) + colType := colTypeMap[col] + if colType == querypb.Type_NULL_TYPE { + log.Errorf("vtexplain: invalid column %s, typeMap +%v", col, colTypeMap) + } + colNames = append(colNames, col) + colTypes = append(colTypes, colType) + case *sqlparser.FuncExpr: + // As a shortcut, functions are integral types + colNames = append(colNames, sqlparser.String(node)) + colTypes = append(colTypes, querypb.Type_INT32) + case *sqlparser.SQLVal: + colNames = append(colNames, sqlparser.String(node)) + switch node.Type { + case sqlparser.IntVal: + fallthrough + case sqlparser.HexNum: + fallthrough + case sqlparser.HexVal: + fallthrough + case sqlparser.BitVal: + colTypes = append(colTypes, querypb.Type_INT32) + case sqlparser.StrVal: + colTypes = append(colTypes, querypb.Type_VARCHAR) + case sqlparser.FloatVal: + colTypes = append(colTypes, querypb.Type_FLOAT64) + default: + log.Errorf("vtexplain: unsupported sql value %s", sqlparser.String(node)) + } + case *sqlparser.ParenExpr: + colNames, colTypes = inferColTypeFromExpr(node.Expr, colTypeMap, colNames, colTypes) + case *sqlparser.CaseExpr: + colNames, colTypes = inferColTypeFromExpr(node.Whens[0].Val, colTypeMap, colNames, colTypes) + default: + log.Errorf("vtexplain: unsupported select expression type +%v node %s", reflect.TypeOf(node), sqlparser.String(node)) + } + + return colNames, colTypes +} From 8de216c8037e25bae680999f7e3fd75e78a28852 Mon Sep 17 00:00:00 2001 From: Michael Demmer Date: Tue, 27 Aug 2019 10:24:28 -0700 Subject: [PATCH 2/3] add vtexplain support for union statements Signed-off-by: Michael Demmer --- .../testdata/multi-output/selectsharded-output.txt | 5 +++++ go/vt/vtexplain/testdata/selectsharded-queries.sql | 2 ++ go/vt/vtexplain/vtexplain_vttablet.go | 10 +++++++++- 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/go/vt/vtexplain/testdata/multi-output/selectsharded-output.txt b/go/vt/vtexplain/testdata/multi-output/selectsharded-output.txt index 198307bed3..43ef9e6e10 100644 --- a/go/vt/vtexplain/testdata/multi-output/selectsharded-output.txt +++ b/go/vt/vtexplain/testdata/multi-output/selectsharded-output.txt @@ -161,3 +161,8 @@ select id, case when substr(name, 1, 5) = 'alice' then 'ALICE' when name = 'bob' 1 ks_sharded/-40: select id, case when substr(name, 1, 5) = 'alice' then 'ALICE' when name = 'bob' then 'BOB' else 'OTHER' end as name from user where id = 1 limit 10001 /* select case */ ---------------------------------------------------------------------- +select id, 'abc' as test from user where id = 1 union all select id, 'def' as test from user where id = 1 /* union all */ + +1 ks_sharded/-40: select id, 'abc' as test from user where id = 1 union all select id, 'def' as test from user where id = 1 limit 10001 /* union all */ + +---------------------------------------------------------------------- diff --git a/go/vt/vtexplain/testdata/selectsharded-queries.sql b/go/vt/vtexplain/testdata/selectsharded-queries.sql index 6c3f7476f0..ccee352396 100644 --- a/go/vt/vtexplain/testdata/selectsharded-queries.sql +++ b/go/vt/vtexplain/testdata/selectsharded-queries.sql @@ -30,3 +30,5 @@ select id, substring_index(name, '123456', -1) from user where id = 123 /* selec select id, case when name = 'alice' then 'ALICE' when name = 'bob' then 'BOB' end as name from user where id = 1 /* select case */; select id, case when name = 'alice' then 'ALICE' when name = 'bob' then 'BOB' else 'OTHER' end as name from user where id = 1 /* select case */; select id, case when substr(name, 1, 5) = 'alice' then 'ALICE' when name = 'bob' then 'BOB' else 'OTHER' end as name from user where id = 1 /* select case */; + +select id, 'abc' as test from user where id = 1 union all select id, 'def' as test from user where id = 1 /* union all */; \ No newline at end of file diff --git a/go/vt/vtexplain/vtexplain_vttablet.go b/go/vt/vtexplain/vtexplain_vttablet.go index e45a822d37..1dc67537a8 100644 --- a/go/vt/vtexplain/vtexplain_vttablet.go +++ b/go/vt/vtexplain/vtexplain_vttablet.go @@ -502,7 +502,15 @@ func (t *explainTablet) HandleQuery(c *mysql.Conn, query string, callback func(* return err } - selStmt := stmt.(*sqlparser.Select) + var selStmt *sqlparser.Select + switch stmt.(type) { + case *sqlparser.Select: + selStmt = stmt.(*sqlparser.Select) + case *sqlparser.Union: + selStmt = stmt.(*sqlparser.Union).Left.(*sqlparser.Select) + default: + return fmt.Errorf("vtexplain: unsupported statement type +%v", reflect.TypeOf(stmt)) + } if len(selStmt.From) != 1 { return fmt.Errorf("unsupported select with multiple from clauses") From f3ba59e6861ddb0c8d602f17b21a7c1d82624add Mon Sep 17 00:00:00 2001 From: Michael Demmer Date: Tue, 27 Aug 2019 10:27:08 -0700 Subject: [PATCH 3/3] make union all work for three or more statements Signed-off-by: Michael Demmer --- .../vtexplain/testdata/multi-output/selectsharded-output.txt | 4 ++-- go/vt/vtexplain/testdata/selectsharded-queries.sql | 2 +- go/vt/vtexplain/vtexplain_vttablet.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go/vt/vtexplain/testdata/multi-output/selectsharded-output.txt b/go/vt/vtexplain/testdata/multi-output/selectsharded-output.txt index 43ef9e6e10..fa6637e58c 100644 --- a/go/vt/vtexplain/testdata/multi-output/selectsharded-output.txt +++ b/go/vt/vtexplain/testdata/multi-output/selectsharded-output.txt @@ -161,8 +161,8 @@ select id, case when substr(name, 1, 5) = 'alice' then 'ALICE' when name = 'bob' 1 ks_sharded/-40: select id, case when substr(name, 1, 5) = 'alice' then 'ALICE' when name = 'bob' then 'BOB' else 'OTHER' end as name from user where id = 1 limit 10001 /* select case */ ---------------------------------------------------------------------- -select id, 'abc' as test from user where id = 1 union all select id, 'def' as test from user where id = 1 /* union all */ +select id, 'abc' as test from user where id = 1 union all select id, 'def' as test from user where id = 1 union all select id, 'ghi' as test from user where id = 1 /* union all */ -1 ks_sharded/-40: select id, 'abc' as test from user where id = 1 union all select id, 'def' as test from user where id = 1 limit 10001 /* union all */ +1 ks_sharded/-40: select id, 'abc' as test from user where id = 1 union all select id, 'def' as test from user where id = 1 union all select id, 'ghi' as test from user where id = 1 limit 10001 /* union all */ ---------------------------------------------------------------------- diff --git a/go/vt/vtexplain/testdata/selectsharded-queries.sql b/go/vt/vtexplain/testdata/selectsharded-queries.sql index ccee352396..b9c42e3f05 100644 --- a/go/vt/vtexplain/testdata/selectsharded-queries.sql +++ b/go/vt/vtexplain/testdata/selectsharded-queries.sql @@ -31,4 +31,4 @@ select id, case when name = 'alice' then 'ALICE' when name = 'bob' then 'BOB' en select id, case when name = 'alice' then 'ALICE' when name = 'bob' then 'BOB' else 'OTHER' end as name from user where id = 1 /* select case */; select id, case when substr(name, 1, 5) = 'alice' then 'ALICE' when name = 'bob' then 'BOB' else 'OTHER' end as name from user where id = 1 /* select case */; -select id, 'abc' as test from user where id = 1 union all select id, 'def' as test from user where id = 1 /* union all */; \ No newline at end of file +select id, 'abc' as test from user where id = 1 union all select id, 'def' as test from user where id = 1 union all select id, 'ghi' as test from user where id = 1 /* union all */; \ No newline at end of file diff --git a/go/vt/vtexplain/vtexplain_vttablet.go b/go/vt/vtexplain/vtexplain_vttablet.go index 1dc67537a8..d16b94ea7e 100644 --- a/go/vt/vtexplain/vtexplain_vttablet.go +++ b/go/vt/vtexplain/vtexplain_vttablet.go @@ -507,7 +507,7 @@ func (t *explainTablet) HandleQuery(c *mysql.Conn, query string, callback func(* case *sqlparser.Select: selStmt = stmt.(*sqlparser.Select) case *sqlparser.Union: - selStmt = stmt.(*sqlparser.Union).Left.(*sqlparser.Select) + selStmt = stmt.(*sqlparser.Union).Right.(*sqlparser.Select) default: return fmt.Errorf("vtexplain: unsupported statement type +%v", reflect.TypeOf(stmt)) }