From 67a82405f0dfc5058ec26ac50e8e215bcc22770e Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Tue, 11 Aug 2015 14:40:59 -0700 Subject: [PATCH 01/80] upsert: Make value & last_insert_id as keywords vttablet cannot support those two functions in upserts. Marking them as keywords automatically prevents their use. This way, we don't have to explicitly look for these keywords inside upserts to prevent them. If/when we decide to support them, we can add these keywords to the keyword_as_func rule in the parser. --- data/test/sqlparser_test/parse_fail.sql | 2 + go/vt/sqlparser/sql.go | 810 ++++++++++++------------ go/vt/sqlparser/sql.y | 1 + go/vt/sqlparser/token.go | 142 +++-- 4 files changed, 484 insertions(+), 471 deletions(-) diff --git a/data/test/sqlparser_test/parse_fail.sql b/data/test/sqlparser_test/parse_fail.sql index 60775c4a05..ae92ff7427 100644 --- a/data/test/sqlparser_test/parse_fail.sql +++ b/data/test/sqlparser_test/parse_fail.sql @@ -11,6 +11,8 @@ select * from t where :1 = 2#syntax error at position 24 near : select * from t where :. = 2#syntax error at position 24 near : select * from t where ::1 = 2#syntax error at position 25 near :: select * from t where ::. = 2#syntax error at position 25 near :: +update a set c = value(1)#syntax error at position 23 near value +update a set c = last_insert_id(1)#syntax error at position 32 near last_insert_id select(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(#max nesting level reached at position 406 select(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(#syntax error at position 405 select /* aa#syntax error at position 13 near /* aa diff --git a/go/vt/sqlparser/sql.go b/go/vt/sqlparser/sql.go index 2f7f0cd20d..3d928afa77 100644 --- a/go/vt/sqlparser/sql.go +++ b/go/vt/sqlparser/sql.go @@ -102,58 +102,60 @@ const DEFAULT = 57374 const SET = 57375 const LOCK = 57376 const KEYRANGE = 57377 -const ID = 57378 -const STRING = 57379 -const NUMBER = 57380 -const VALUE_ARG = 57381 -const LIST_ARG = 57382 -const COMMENT = 57383 -const LE = 57384 -const GE = 57385 -const NE = 57386 -const NULL_SAFE_EQUAL = 57387 -const UNION = 57388 -const MINUS = 57389 -const EXCEPT = 57390 -const INTERSECT = 57391 -const JOIN = 57392 -const STRAIGHT_JOIN = 57393 -const LEFT = 57394 -const RIGHT = 57395 -const INNER = 57396 -const OUTER = 57397 -const CROSS = 57398 -const NATURAL = 57399 -const USE = 57400 -const FORCE = 57401 -const ON = 57402 -const OR = 57403 -const AND = 57404 -const NOT = 57405 -const SHIFT_LEFT = 57406 -const SHIFT_RIGHT = 57407 -const UNARY = 57408 -const CASE = 57409 -const WHEN = 57410 -const THEN = 57411 -const ELSE = 57412 -const END = 57413 -const CREATE = 57414 -const ALTER = 57415 -const DROP = 57416 -const RENAME = 57417 -const ANALYZE = 57418 -const TABLE = 57419 -const INDEX = 57420 -const VIEW = 57421 -const TO = 57422 -const IGNORE = 57423 -const IF = 57424 -const UNIQUE = 57425 -const USING = 57426 -const SHOW = 57427 -const DESCRIBE = 57428 -const EXPLAIN = 57429 +const VALUE = 57378 +const LAST_INSERT_ID = 57379 +const ID = 57380 +const STRING = 57381 +const NUMBER = 57382 +const VALUE_ARG = 57383 +const LIST_ARG = 57384 +const COMMENT = 57385 +const LE = 57386 +const GE = 57387 +const NE = 57388 +const NULL_SAFE_EQUAL = 57389 +const UNION = 57390 +const MINUS = 57391 +const EXCEPT = 57392 +const INTERSECT = 57393 +const JOIN = 57394 +const STRAIGHT_JOIN = 57395 +const LEFT = 57396 +const RIGHT = 57397 +const INNER = 57398 +const OUTER = 57399 +const CROSS = 57400 +const NATURAL = 57401 +const USE = 57402 +const FORCE = 57403 +const ON = 57404 +const OR = 57405 +const AND = 57406 +const NOT = 57407 +const SHIFT_LEFT = 57408 +const SHIFT_RIGHT = 57409 +const UNARY = 57410 +const CASE = 57411 +const WHEN = 57412 +const THEN = 57413 +const ELSE = 57414 +const END = 57415 +const CREATE = 57416 +const ALTER = 57417 +const DROP = 57418 +const RENAME = 57419 +const ANALYZE = 57420 +const TABLE = 57421 +const INDEX = 57422 +const VIEW = 57423 +const TO = 57424 +const IGNORE = 57425 +const IF = 57426 +const UNIQUE = 57427 +const USING = 57428 +const SHOW = 57429 +const DESCRIBE = 57430 +const EXPLAIN = 57431 var yyToknames = [...]string{ "$end", @@ -191,6 +193,8 @@ var yyToknames = [...]string{ "SET", "LOCK", "KEYRANGE", + "VALUE", + "LAST_INSERT_ID", "ID", "STRING", "NUMBER", @@ -272,7 +276,7 @@ var yyExca = [...]int{ 1, -1, -2, 0, -1, 68, - 81, 203, + 83, 203, -2, 202, } @@ -282,133 +286,137 @@ const yyPrivate = 57344 var yyTokenNames []string var yyStates []string -const yyLast = 691 +const yyLast = 732 var yyAct = [...]int{ 98, 166, 163, 340, 236, 373, 257, 93, 303, 63, 94, 215, 295, 204, 92, 165, 3, 382, 184, 132, - 64, 248, 237, 138, 137, 82, 239, 83, 301, 192, - 78, 101, 108, 70, 264, 43, 107, 44, 66, 110, - 126, 72, 351, 53, 75, 65, 102, 68, 104, 105, - 106, 46, 47, 48, 101, 320, 322, 103, 88, 107, - 237, 350, 110, 349, 71, 74, 237, 87, 237, 102, - 68, 104, 105, 106, 49, 237, 45, 237, 249, 90, - 103, 130, 95, 96, 84, 38, 134, 40, 321, 331, - 97, 41, 147, 111, 167, 249, 279, 293, 168, 170, - 171, 136, 90, 119, 115, 95, 96, 121, 109, 28, - 29, 30, 31, 97, 178, 66, 111, 137, 66, 346, - 188, 187, 65, 296, 182, 65, 268, 269, 270, 271, - 272, 109, 273, 274, 174, 88, 210, 188, 186, 260, - 50, 73, 214, 129, 224, 222, 223, 348, 226, 227, + 64, 248, 138, 137, 237, 82, 192, 83, 78, 239, + 301, 70, 108, 38, 43, 40, 44, 14, 66, 41, + 264, 72, 126, 53, 75, 65, 351, 268, 269, 270, + 271, 272, 350, 273, 274, 349, 74, 107, 88, 237, + 110, 71, 49, 320, 322, 45, 237, 87, 237, 331, + 68, 104, 105, 106, 28, 29, 30, 31, 237, 237, + 103, 130, 46, 47, 48, 249, 134, 293, 249, 279, + 136, 138, 137, 119, 167, 237, 321, 115, 168, 170, + 171, 147, 153, 154, 147, 95, 96, 198, 121, 333, + 73, 138, 137, 97, 178, 66, 111, 224, 66, 137, + 188, 187, 65, 346, 182, 65, 196, 237, 296, 260, + 117, 109, 199, 129, 174, 88, 210, 188, 186, 314, + 50, 296, 214, 348, 315, 222, 223, 211, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 162, 164, - 209, 198, 237, 347, 208, 169, 51, 52, 138, 137, - 238, 240, 241, 217, 237, 88, 88, 242, 211, 318, - 196, 66, 66, 117, 199, 117, 333, 225, 65, 255, - 253, 246, 59, 261, 296, 252, 138, 137, 243, 245, - 317, 256, 153, 154, 147, 212, 213, 150, 151, 152, - 153, 154, 147, 123, 316, 131, 125, 239, 280, 241, - 278, 91, 265, 282, 283, 77, 268, 269, 270, 271, - 272, 281, 273, 274, 185, 314, 195, 197, 194, 208, - 315, 312, 286, 358, 335, 290, 313, 88, 287, 118, - 289, 113, 217, 259, 103, 116, 91, 91, 112, 300, - 132, 189, 298, 302, 172, 173, 299, 175, 176, 292, - 288, 202, 185, 180, 80, 360, 361, 277, 266, 181, - 310, 311, 61, 324, 73, 326, 103, 328, 28, 29, - 30, 31, 103, 329, 61, 68, 332, 206, 91, 208, - 208, 14, 66, 91, 91, 330, 216, 338, 341, 336, - 61, 337, 14, 15, 16, 17, 117, 294, 146, 145, - 148, 149, 150, 151, 152, 153, 154, 147, 135, 122, - 352, 218, 61, 79, 262, 353, 354, 103, 91, 91, - 18, 114, 103, 355, 334, 73, 356, 364, 241, 58, + 209, 347, 225, 312, 208, 169, 51, 52, 313, 318, + 238, 240, 241, 217, 317, 88, 88, 242, 59, 316, + 117, 66, 66, 239, 195, 197, 194, 131, 65, 255, + 253, 246, 185, 261, 185, 252, 358, 335, 243, 245, + 290, 256, 118, 103, 61, 212, 213, 150, 151, 152, + 153, 154, 147, 123, 103, 112, 125, 73, 280, 241, + 278, 91, 265, 282, 283, 68, 268, 269, 270, 271, + 272, 281, 273, 274, 132, 77, 180, 113, 266, 208, + 117, 116, 286, 28, 29, 30, 31, 88, 287, 14, + 289, 103, 217, 259, 218, 61, 91, 91, 277, 300, + 103, 189, 298, 302, 172, 173, 299, 175, 176, 292, + 288, 202, 122, 360, 361, 107, 135, 61, 79, 181, + 310, 311, 61, 324, 80, 326, 114, 328, 334, 104, + 105, 106, 103, 329, 379, 73, 332, 206, 91, 208, + 208, 355, 66, 91, 91, 330, 216, 338, 341, 336, + 14, 337, 380, 58, 285, 386, 190, 294, 146, 145, + 148, 149, 150, 151, 152, 153, 154, 147, 56, 219, + 352, 220, 221, 251, 262, 353, 354, 128, 91, 91, + 54, 304, 345, 305, 258, 344, 356, 364, 241, 309, 362, 91, 148, 149, 150, 151, 152, 153, 154, 147, - 370, 341, 371, 14, 379, 285, 386, 374, 374, 374, - 66, 372, 206, 375, 376, 67, 190, 65, 381, 377, - 383, 384, 380, 387, 128, 216, 251, 388, 107, 389, - 219, 56, 220, 221, 19, 20, 22, 21, 23, 54, - 104, 105, 106, 304, 345, 305, 258, 24, 25, 26, - 91, 60, 363, 344, 365, 91, 309, 185, 62, 385, - 369, 76, 14, 33, 275, 81, 133, 191, 39, 263, - 193, 86, 206, 206, 60, 42, 69, 254, 60, 244, - 179, 101, 378, 359, 339, 120, 107, 342, 343, 110, - 124, 308, 291, 127, 357, 177, 102, 68, 104, 105, - 106, 247, 100, 99, 297, 14, 250, 103, 139, 146, - 145, 148, 149, 150, 151, 152, 153, 154, 147, 89, - 101, 319, 205, 267, 203, 107, 85, 55, 110, 90, - 27, 183, 95, 96, 84, 102, 68, 104, 105, 106, - 97, 57, 200, 111, 32, 201, 103, 207, 86, 13, - 91, 12, 91, 14, 11, 366, 367, 368, 109, 10, - 34, 35, 36, 37, 237, 9, 8, 7, 90, 6, - 5, 95, 96, 107, 4, 2, 110, 1, 0, 97, - 0, 0, 111, 0, 68, 104, 105, 106, 86, 86, - 107, 0, 0, 110, 103, 0, 0, 109, 0, 0, - 0, 68, 104, 105, 106, 0, 0, 0, 0, 0, - 0, 103, 0, 0, 0, 0, 0, 0, 0, 95, - 96, 276, 207, 0, 0, 0, 0, 97, 0, 0, - 111, 0, 0, 0, 0, 0, 95, 96, 0, 0, - 0, 0, 0, 0, 97, 109, 327, 111, 146, 145, - 148, 149, 150, 151, 152, 153, 154, 147, 0, 0, - 86, 0, 109, 140, 144, 142, 143, 145, 148, 149, - 150, 151, 152, 153, 154, 147, 306, 0, 0, 307, - 0, 0, 207, 207, 158, 159, 160, 161, 0, 155, - 156, 157, 0, 323, 284, 325, 146, 145, 148, 149, - 150, 151, 152, 153, 154, 147, 0, 0, 0, 0, - 141, 146, 145, 148, 149, 150, 151, 152, 153, 154, - 147, 146, 145, 148, 149, 150, 151, 152, 153, 154, - 147, + 370, 341, 371, 185, 62, 385, 369, 374, 374, 374, + 66, 372, 206, 375, 376, 67, 14, 65, 381, 377, + 383, 384, 33, 387, 275, 216, 133, 388, 191, 389, + 39, 14, 15, 16, 17, 146, 145, 148, 149, 150, + 151, 152, 153, 154, 147, 263, 244, 193, 101, 42, + 91, 60, 363, 107, 365, 91, 110, 69, 254, 18, + 179, 76, 378, 102, 32, 81, 68, 104, 105, 106, + 359, 86, 206, 206, 60, 339, 103, 343, 60, 308, + 34, 35, 36, 37, 291, 120, 177, 342, 247, 100, + 124, 99, 297, 127, 250, 139, 89, 319, 90, 205, + 267, 95, 96, 84, 203, 85, 55, 27, 57, 97, + 13, 12, 111, 11, 10, 19, 20, 22, 21, 23, + 9, 107, 8, 7, 110, 6, 5, 109, 24, 25, + 26, 183, 4, 237, 68, 104, 105, 106, 2, 1, + 0, 0, 200, 0, 103, 201, 101, 207, 86, 0, + 91, 107, 91, 0, 110, 366, 367, 368, 0, 0, + 0, 102, 0, 0, 68, 104, 105, 106, 0, 95, + 96, 0, 0, 0, 103, 0, 0, 97, 0, 0, + 111, 0, 0, 14, 0, 0, 0, 0, 86, 86, + 0, 0, 0, 0, 0, 109, 90, 0, 101, 95, + 96, 84, 0, 107, 0, 0, 110, 97, 0, 0, + 111, 0, 0, 102, 0, 0, 68, 104, 105, 106, + 0, 276, 207, 0, 0, 109, 103, 0, 0, 0, + 0, 0, 0, 0, 0, 101, 0, 0, 0, 0, + 107, 0, 0, 110, 0, 0, 0, 0, 90, 357, + 102, 95, 96, 68, 104, 105, 106, 0, 0, 97, + 86, 0, 111, 103, 146, 145, 148, 149, 150, 151, + 152, 153, 154, 147, 0, 0, 306, 109, 0, 307, + 0, 0, 207, 207, 0, 90, 0, 0, 95, 96, + 0, 0, 0, 323, 0, 325, 97, 0, 0, 111, + 140, 144, 142, 143, 145, 148, 149, 150, 151, 152, + 153, 154, 147, 0, 109, 0, 0, 0, 0, 0, + 0, 0, 0, 158, 159, 160, 161, 0, 155, 156, + 157, 327, 0, 146, 145, 148, 149, 150, 151, 152, + 153, 154, 147, 0, 0, 0, 0, 0, 0, 141, + 146, 145, 148, 149, 150, 151, 152, 153, 154, 147, + 284, 0, 146, 145, 148, 149, 150, 151, 152, 153, + 154, 147, } var yyPact = [...]int{ - 307, -1000, -1000, 238, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -7, - -59, -16, -41, -18, -1000, -1000, -1000, 417, 382, -1000, - -1000, -1000, 373, -1000, 320, 274, 409, 259, -64, -29, - 248, -1000, -27, 248, -1000, 274, -67, 297, -67, 274, - -1000, -1000, -1000, -1000, -1000, 11, -1000, 217, 274, 308, - 23, -1000, 274, 131, -1000, 202, -1000, 22, -1000, 274, - 39, 293, -1000, -1000, 274, -1000, -55, 274, 364, 78, - 248, -1000, 206, -1000, -1000, 309, 20, 130, 602, -1000, - 34, 460, -1000, -1000, -1000, 525, 525, 525, 208, 208, - -1000, 208, 208, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, 525, -1000, 240, 259, 274, 407, 259, 525, 248, - -1000, 356, -70, -1000, 148, -1000, 274, -1000, -1000, 274, - -1000, 246, 11, -1000, -1000, 248, 105, 34, 34, 525, - 291, 369, 525, 525, 119, 525, 525, 525, 525, 525, - 525, 525, 525, 525, 525, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, 602, -43, 59, -28, 602, -1000, 508, - -1000, -1000, 421, 11, -1000, 417, 363, -5, 612, 358, - 259, 259, 262, -1000, 393, 34, -1000, 612, -1000, -1000, - -1000, 74, 248, -1000, -61, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, 224, 171, 258, 296, 15, -1000, -1000, - -1000, -1000, -1000, 50, 612, -1000, 508, -1000, -1000, 291, - 525, 525, 612, 587, -1000, 340, 281, 557, -1000, 134, - 134, 126, 126, 126, 14, 14, -1000, -1000, -1000, 525, - -1000, 612, -1000, -35, 11, -35, 191, 12, -1000, 34, - 58, 208, 238, 129, -26, -1000, 393, 388, 391, 130, - 274, -1000, -1000, 274, -1000, 405, 246, 246, -1000, -1000, - 186, 180, 159, 145, 124, -8, -1000, 274, 71, 274, - -28, -1000, 612, 539, 525, -1000, 612, -1000, -35, -1000, - 363, 3, -1000, 525, 102, -1000, 314, 190, -1000, -1000, - -1000, 259, 388, -1000, 525, 525, -1000, -1000, 401, 390, - 171, 54, -1000, 108, -1000, 92, -1000, -1000, -1000, -1000, - -30, -32, -51, -1000, -1000, -1000, -1000, 525, 612, -1000, - -81, -1000, 612, 525, 312, 208, -1000, -1000, 400, 189, - -1000, 249, -1000, 393, 34, 525, 34, -1000, -1000, 208, - 208, 208, 612, -1000, 612, 413, -1000, 525, 525, -1000, - -1000, -1000, 388, 130, 163, 130, 248, 248, 248, 259, - 612, -1000, 348, -37, -1000, -37, -37, 131, -1000, 412, - 345, -1000, 248, -1000, -1000, -1000, 248, -1000, 248, -1000, + 386, -1000, -1000, 191, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -61, + -62, -29, -12, -32, -1000, -1000, -1000, 371, 323, -1000, + -1000, -1000, 310, -1000, 284, 217, 355, 187, -68, -34, + 179, -1000, -38, 179, -1000, 217, -71, 240, -71, 217, + -1000, -1000, -1000, -1000, -1000, 486, -1000, 172, 217, 253, + 14, -1000, 217, 124, -1000, 153, -1000, 10, -1000, 217, + 38, 234, -1000, -1000, 217, -1000, -55, 217, 317, 66, + 179, -1000, 178, -1000, -1000, 257, 7, 43, 639, -1000, + 575, 538, -1000, -1000, -1000, 456, 456, 456, 155, 155, + -1000, 155, 155, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, 456, -1000, 203, 187, 217, 353, 187, 456, 179, + -1000, 296, -75, -1000, 94, -1000, 217, -1000, -1000, 217, + -1000, 166, 486, -1000, -1000, 179, 72, 575, 575, 456, + 212, 308, 456, 456, 92, 456, 456, 456, 456, 456, + 456, 456, 456, 456, 456, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, 639, -46, 22, -27, 639, -1000, 32, + -1000, -1000, 388, 486, -1000, 371, 250, 3, 324, 305, + 187, 187, 184, -1000, 331, 575, -1000, 324, -1000, -1000, + -1000, 62, 179, -1000, -57, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, 182, 169, 239, 244, 6, -1000, -1000, + -1000, -1000, -1000, 50, 324, -1000, 32, -1000, -1000, 212, + 456, 456, 324, 651, -1000, 289, 279, 592, -1000, 132, + 132, 24, 24, 24, 21, 21, -1000, -1000, -1000, 456, + -1000, 324, -1000, -37, 486, -37, 144, 0, -1000, 575, + 61, 155, 191, 74, -26, -1000, 331, 326, 329, 43, + 217, -1000, -1000, 217, -1000, 338, 166, 166, -1000, -1000, + 106, 82, 122, 117, 112, -2, -1000, 217, -10, 217, + -27, -1000, 324, 622, 456, -1000, 324, -1000, -37, -1000, + 250, -19, -1000, 456, 23, -1000, 258, 141, -1000, -1000, + -1000, 187, 326, -1000, 456, 456, -1000, -1000, 333, 328, + 169, 56, -1000, 104, -1000, 86, -1000, -1000, -1000, -1000, + -40, -43, -49, -1000, -1000, -1000, -1000, 456, 324, -1000, + -81, -1000, 324, 456, 270, 155, -1000, -1000, 553, 140, + -1000, 247, -1000, 331, 575, 456, 575, -1000, -1000, 155, + 155, 155, 324, -1000, 324, 359, -1000, 456, 456, -1000, + -1000, -1000, 326, 43, 127, 43, 179, 179, 179, 187, + 324, -1000, 278, -39, -1000, -39, -39, 124, -1000, 358, + 294, -1000, 179, -1000, -1000, -1000, 179, -1000, 179, -1000, } var yyPgo = [...]int{ - 0, 537, 535, 15, 534, 530, 529, 527, 526, 525, - 519, 514, 511, 509, 504, 501, 490, 487, 25, 27, - 486, 484, 13, 483, 482, 192, 481, 5, 18, 67, - 479, 468, 466, 14, 2, 11, 1, 464, 10, 463, - 32, 7, 462, 461, 21, 455, 452, 451, 448, 6, - 444, 3, 443, 8, 442, 440, 437, 12, 9, 20, - 225, 436, 435, 430, 429, 428, 427, 0, 426, 375, - 424, 140, 423, 165, 4, + 0, 499, 498, 15, 492, 486, 485, 483, 482, 480, + 474, 473, 471, 470, 424, 468, 467, 466, 25, 27, + 465, 464, 13, 460, 459, 178, 457, 5, 18, 67, + 456, 455, 454, 14, 2, 11, 1, 452, 10, 451, + 32, 7, 449, 448, 21, 446, 444, 439, 437, 6, + 435, 3, 430, 8, 422, 420, 418, 12, 9, 20, + 235, 417, 409, 407, 405, 390, 388, 0, 386, 375, + 384, 140, 382, 165, 4, } var yyR1 = [...]int{ @@ -461,44 +469,44 @@ var yyR2 = [...]int{ var yyChk = [...]int{ -1000, -1, -2, -3, -4, -5, -6, -7, -8, -9, - -10, -11, -12, -13, 5, 6, 7, 8, 33, 87, - 88, 90, 89, 91, 100, 101, 102, -16, 50, 51, - 52, 53, -14, -72, -14, -14, -14, -14, 92, -65, - 94, 98, -62, 94, 96, 92, 92, 93, 94, 92, + -10, -11, -12, -13, 5, 6, 7, 8, 33, 89, + 90, 92, 91, 93, 102, 103, 104, -16, 52, 53, + 54, 55, -14, -72, -14, -14, -14, -14, 94, -65, + 96, 100, -62, 96, 98, 94, 94, 95, 96, 94, -71, -71, -71, -3, 17, -17, 18, -15, 29, -25, - -69, 36, 9, -58, -59, -41, -67, -69, 36, -61, - 97, 93, -67, 36, 92, -67, -69, -60, 97, 36, - -60, -69, -18, -19, 73, -20, -69, -29, -34, -30, - 68, -73, -33, -41, -38, 71, 72, 79, -67, -39, - -42, 20, 35, 46, 37, 38, 39, 25, -40, 97, - 28, 82, 41, -25, 33, 81, -25, 54, 47, 81, - -69, 68, 36, -71, -69, -71, 95, -69, 20, 65, - -67, 9, 54, -68, -67, 19, 81, 67, 66, -31, - 21, 68, 23, 24, 22, 70, 69, 78, 71, 72, - 73, 74, 75, 76, 77, 47, 48, 49, 42, 43, - 44, 45, -29, -34, -29, -3, -36, -34, -34, -73, + -69, 38, 9, -58, -59, -41, -67, -69, 38, -61, + 99, 95, -67, 38, 94, -67, -69, -60, 99, 38, + -60, -69, -18, -19, 75, -20, -69, -29, -34, -30, + 70, -73, -33, -41, -38, 73, 74, 81, -67, -39, + -42, 20, 35, 48, 39, 40, 41, 25, -40, 99, + 28, 84, 43, -25, 33, 83, -25, 56, 49, 83, + -69, 70, 38, -71, -69, -71, 97, -69, 20, 67, + -67, 9, 56, -68, -67, 19, 83, 69, 68, -31, + 21, 70, 23, 24, 22, 72, 71, 80, 73, 74, + 75, 76, 77, 78, 79, 49, 50, 51, 44, 45, + 46, 47, -29, -34, -29, -3, -36, -34, -34, -73, -34, -34, -73, -73, -40, -73, -73, -45, -34, -55, 33, -73, -58, -69, -28, 10, -59, -34, -67, -71, - 20, -66, 99, -63, 90, 88, 32, 89, 13, 36, + 20, -66, 101, -63, 92, 90, 32, 91, 13, 38, -69, -69, -71, -21, -22, -24, -73, -69, -40, -19, - -67, 73, -29, -29, -34, -35, -73, -40, 40, 21, - 23, 24, -34, -34, 25, 68, -34, -34, -34, -34, - -34, -34, -34, -34, -34, -34, -74, 103, -74, 54, - -74, -34, -74, -18, 18, -18, -33, -43, -44, 83, + -67, 75, -29, -29, -34, -35, -73, -40, 42, 21, + 23, 24, -34, -34, 25, 70, -34, -34, -34, -34, + -34, -34, -34, -34, -34, -34, -74, 105, -74, 56, + -74, -34, -74, -18, 18, -18, -33, -43, -44, 85, -32, 28, -3, -58, -56, -41, -28, -49, 13, -29, - 65, -67, -71, -64, 95, -28, 54, -23, 55, 56, - 57, 58, 59, 61, 62, -70, -69, 19, -22, 81, - -36, -35, -34, -34, 67, 25, -34, -74, -18, -74, - 54, -46, -44, 85, -29, -57, 65, -37, -38, -57, - -74, 54, -49, -53, 15, 14, -69, -69, -47, 11, - -22, -22, 55, 60, 55, 60, 55, 55, 55, -26, - 63, 96, 64, -69, -74, -69, -74, 67, -34, -74, - -33, 86, -34, 84, 30, 54, -41, -53, -34, -50, - -51, -34, -71, -48, 12, 14, 65, 55, 55, 93, - 93, 93, -34, -74, -34, 31, -38, 54, 54, -52, + 67, -67, -71, -64, 97, -28, 56, -23, 57, 58, + 59, 60, 61, 63, 64, -70, -69, 19, -22, 83, + -36, -35, -34, -34, 69, 25, -34, -74, -18, -74, + 56, -46, -44, 87, -29, -57, 67, -37, -38, -57, + -74, 56, -49, -53, 15, 14, -69, -69, -47, 11, + -22, -22, 57, 62, 57, 62, 57, 57, 57, -26, + 65, 98, 66, -69, -74, -69, -74, 69, -34, -74, + -33, 88, -34, 86, 30, 56, -41, -53, -34, -50, + -51, -34, -71, -48, 12, 14, 67, 57, 57, 95, + 95, 95, -34, -74, -34, 31, -38, 56, 56, -52, 26, 27, -49, -29, -36, -29, -73, -73, -73, 7, -34, -51, -53, -27, -67, -27, -27, -58, -54, 16, - 34, -74, 54, -74, -74, 7, 21, -67, -67, -67, + 34, -74, 56, -74, -74, 7, 21, -67, -67, -67, } var yyDef = [...]int{ @@ -547,16 +555,16 @@ var yyTok1 = [...]int{ 1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 3, 3, 3, 3, 3, 3, 3, 75, 70, 3, - 46, 103, 73, 71, 54, 72, 81, 74, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 77, 72, 3, + 48, 105, 75, 73, 56, 74, 83, 76, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 48, 47, 49, 3, 3, 3, 3, 3, 3, 3, + 50, 49, 51, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 3, 3, 3, 3, 78, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 80, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 3, 3, 3, 3, 69, 3, 79, + 3, 3, 3, 3, 71, 3, 81, } var yyTok2 = [...]int{ @@ -564,11 +572,11 @@ var yyTok2 = [...]int{ 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, - 42, 43, 44, 45, 50, 51, 52, 53, 55, 56, + 42, 43, 44, 45, 46, 47, 52, 53, 54, 55, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, - 67, 68, 76, 77, 80, 82, 83, 84, 85, 86, + 67, 68, 69, 70, 78, 79, 82, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, - 97, 98, 99, 100, 101, 102, + 97, 98, 99, 100, 101, 102, 103, 104, } var yyTok3 = [...]int{ 0, @@ -915,37 +923,37 @@ yydefault: case 1: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:160 + //line sql.y:161 { setParseTree(yylex, yyDollar[1].statement) } case 2: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:166 + //line sql.y:167 { yyVAL.statement = yyDollar[1].selStmt } case 13: yyDollar = yyS[yypt-12 : yypt+1] - //line sql.y:182 + //line sql.y:183 { yyVAL.selStmt = &Select{Comments: Comments(yyDollar[2].bytes2), Distinct: yyDollar[3].str, SelectExprs: yyDollar[4].selectExprs, From: yyDollar[6].tableExprs, Where: NewWhere(AST_WHERE, yyDollar[7].boolExpr), GroupBy: GroupBy(yyDollar[8].valExprs), Having: NewWhere(AST_HAVING, yyDollar[9].boolExpr), OrderBy: yyDollar[10].orderBy, Limit: yyDollar[11].limit, Lock: yyDollar[12].str} } case 14: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:186 + //line sql.y:187 { yyVAL.selStmt = &Union{Type: yyDollar[2].str, Left: yyDollar[1].selStmt, Right: yyDollar[3].selStmt} } case 15: yyDollar = yyS[yypt-7 : yypt+1] - //line sql.y:192 + //line sql.y:193 { yyVAL.statement = &Insert{Comments: Comments(yyDollar[2].bytes2), Table: yyDollar[4].tableName, Columns: yyDollar[5].columns, Rows: yyDollar[6].insRows, OnDup: OnDup(yyDollar[7].updateExprs)} } case 16: yyDollar = yyS[yypt-7 : yypt+1] - //line sql.y:196 + //line sql.y:197 { cols := make(Columns, 0, len(yyDollar[6].updateExprs)) vals := make(ValTuple, 0, len(yyDollar[6].updateExprs)) @@ -957,671 +965,671 @@ yydefault: } case 17: yyDollar = yyS[yypt-8 : yypt+1] - //line sql.y:208 + //line sql.y:209 { yyVAL.statement = &Update{Comments: Comments(yyDollar[2].bytes2), Table: yyDollar[3].tableName, Exprs: yyDollar[5].updateExprs, Where: NewWhere(AST_WHERE, yyDollar[6].boolExpr), OrderBy: yyDollar[7].orderBy, Limit: yyDollar[8].limit} } case 18: yyDollar = yyS[yypt-7 : yypt+1] - //line sql.y:214 + //line sql.y:215 { yyVAL.statement = &Delete{Comments: Comments(yyDollar[2].bytes2), Table: yyDollar[4].tableName, Where: NewWhere(AST_WHERE, yyDollar[5].boolExpr), OrderBy: yyDollar[6].orderBy, Limit: yyDollar[7].limit} } case 19: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:220 + //line sql.y:221 { yyVAL.statement = &Set{Comments: Comments(yyDollar[2].bytes2), Exprs: yyDollar[3].updateExprs} } case 20: yyDollar = yyS[yypt-5 : yypt+1] - //line sql.y:226 + //line sql.y:227 { yyVAL.statement = &DDL{Action: AST_CREATE, NewName: yyDollar[4].sqlID} } case 21: yyDollar = yyS[yypt-8 : yypt+1] - //line sql.y:230 + //line sql.y:231 { // Change this to an alter statement yyVAL.statement = &DDL{Action: AST_ALTER, Table: yyDollar[7].sqlID, NewName: yyDollar[7].sqlID} } case 22: yyDollar = yyS[yypt-4 : yypt+1] - //line sql.y:235 + //line sql.y:236 { yyVAL.statement = &DDL{Action: AST_CREATE, NewName: SQLName(yyDollar[3].sqlID)} } case 23: yyDollar = yyS[yypt-6 : yypt+1] - //line sql.y:241 + //line sql.y:242 { yyVAL.statement = &DDL{Action: AST_ALTER, Table: yyDollar[4].sqlID, NewName: yyDollar[4].sqlID} } case 24: yyDollar = yyS[yypt-7 : yypt+1] - //line sql.y:245 + //line sql.y:246 { // Change this to a rename statement yyVAL.statement = &DDL{Action: AST_RENAME, Table: yyDollar[4].sqlID, NewName: yyDollar[7].sqlID} } case 25: yyDollar = yyS[yypt-4 : yypt+1] - //line sql.y:250 + //line sql.y:251 { yyVAL.statement = &DDL{Action: AST_ALTER, Table: SQLName(yyDollar[3].sqlID), NewName: SQLName(yyDollar[3].sqlID)} } case 26: yyDollar = yyS[yypt-5 : yypt+1] - //line sql.y:256 + //line sql.y:257 { yyVAL.statement = &DDL{Action: AST_RENAME, Table: yyDollar[3].sqlID, NewName: yyDollar[5].sqlID} } case 27: yyDollar = yyS[yypt-4 : yypt+1] - //line sql.y:262 + //line sql.y:263 { yyVAL.statement = &DDL{Action: AST_DROP, Table: yyDollar[4].sqlID} } case 28: yyDollar = yyS[yypt-5 : yypt+1] - //line sql.y:266 + //line sql.y:267 { // Change this to an alter statement yyVAL.statement = &DDL{Action: AST_ALTER, Table: yyDollar[5].sqlID, NewName: yyDollar[5].sqlID} } case 29: yyDollar = yyS[yypt-5 : yypt+1] - //line sql.y:271 + //line sql.y:272 { yyVAL.statement = &DDL{Action: AST_DROP, Table: SQLName(yyDollar[4].sqlID)} } case 30: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:277 + //line sql.y:278 { yyVAL.statement = &DDL{Action: AST_ALTER, Table: yyDollar[3].sqlID, NewName: yyDollar[3].sqlID} } case 31: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:283 + //line sql.y:284 { yyVAL.statement = &Other{} } case 32: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:287 + //line sql.y:288 { yyVAL.statement = &Other{} } case 33: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:291 + //line sql.y:292 { yyVAL.statement = &Other{} } case 34: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:296 + //line sql.y:297 { setAllowComments(yylex, true) } case 35: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:300 + //line sql.y:301 { yyVAL.bytes2 = yyDollar[2].bytes2 setAllowComments(yylex, false) } case 36: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:306 + //line sql.y:307 { yyVAL.bytes2 = nil } case 37: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:310 + //line sql.y:311 { yyVAL.bytes2 = append(yyDollar[1].bytes2, yyDollar[2].bytes) } case 38: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:316 + //line sql.y:317 { yyVAL.str = AST_UNION } case 39: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:320 + //line sql.y:321 { yyVAL.str = AST_UNION_ALL } case 40: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:324 + //line sql.y:325 { yyVAL.str = AST_SET_MINUS } case 41: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:328 + //line sql.y:329 { yyVAL.str = AST_EXCEPT } case 42: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:332 + //line sql.y:333 { yyVAL.str = AST_INTERSECT } case 43: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:337 + //line sql.y:338 { yyVAL.str = "" } case 44: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:341 + //line sql.y:342 { yyVAL.str = AST_DISTINCT } case 45: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:347 + //line sql.y:348 { yyVAL.selectExprs = SelectExprs{yyDollar[1].selectExpr} } case 46: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:351 + //line sql.y:352 { yyVAL.selectExprs = append(yyVAL.selectExprs, yyDollar[3].selectExpr) } case 47: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:357 + //line sql.y:358 { yyVAL.selectExpr = &StarExpr{} } case 48: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:361 + //line sql.y:362 { yyVAL.selectExpr = &NonStarExpr{Expr: yyDollar[1].expr, As: yyDollar[2].sqlID} } case 49: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:365 + //line sql.y:366 { yyVAL.selectExpr = &StarExpr{TableName: yyDollar[1].sqlID} } case 50: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:371 + //line sql.y:372 { yyVAL.expr = yyDollar[1].boolExpr } case 51: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:375 + //line sql.y:376 { yyVAL.expr = yyDollar[1].valExpr } case 52: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:380 + //line sql.y:381 { yyVAL.sqlID = "" } case 53: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:384 + //line sql.y:385 { yyVAL.sqlID = yyDollar[1].sqlID } case 54: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:388 + //line sql.y:389 { yyVAL.sqlID = yyDollar[2].sqlID } case 55: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:394 + //line sql.y:395 { yyVAL.tableExprs = TableExprs{yyDollar[1].tableExpr} } case 56: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:398 + //line sql.y:399 { yyVAL.tableExprs = append(yyVAL.tableExprs, yyDollar[3].tableExpr) } case 57: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:404 + //line sql.y:405 { yyVAL.tableExpr = &AliasedTableExpr{Expr: yyDollar[1].smTableExpr, As: yyDollar[2].sqlID, Hints: yyDollar[3].indexHints} } case 58: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:408 + //line sql.y:409 { yyVAL.tableExpr = &ParenTableExpr{Expr: yyDollar[2].tableExpr} } case 59: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:412 + //line sql.y:413 { yyVAL.tableExpr = &JoinTableExpr{LeftExpr: yyDollar[1].tableExpr, Join: yyDollar[2].str, RightExpr: yyDollar[3].tableExpr} } case 60: yyDollar = yyS[yypt-5 : yypt+1] - //line sql.y:416 + //line sql.y:417 { yyVAL.tableExpr = &JoinTableExpr{LeftExpr: yyDollar[1].tableExpr, Join: yyDollar[2].str, RightExpr: yyDollar[3].tableExpr, On: yyDollar[5].boolExpr} } case 61: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:421 + //line sql.y:422 { yyVAL.sqlID = "" } case 62: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:425 + //line sql.y:426 { yyVAL.sqlID = yyDollar[1].sqlID } case 63: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:429 + //line sql.y:430 { yyVAL.sqlID = yyDollar[2].sqlID } case 64: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:435 + //line sql.y:436 { yyVAL.str = AST_JOIN } case 65: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:439 + //line sql.y:440 { yyVAL.str = AST_STRAIGHT_JOIN } case 66: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:443 + //line sql.y:444 { yyVAL.str = AST_LEFT_JOIN } case 67: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:447 + //line sql.y:448 { yyVAL.str = AST_LEFT_JOIN } case 68: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:451 + //line sql.y:452 { yyVAL.str = AST_RIGHT_JOIN } case 69: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:455 + //line sql.y:456 { yyVAL.str = AST_RIGHT_JOIN } case 70: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:459 + //line sql.y:460 { yyVAL.str = AST_JOIN } case 71: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:463 + //line sql.y:464 { yyVAL.str = AST_CROSS_JOIN } case 72: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:467 + //line sql.y:468 { yyVAL.str = AST_NATURAL_JOIN } case 73: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:473 + //line sql.y:474 { yyVAL.smTableExpr = &TableName{Name: yyDollar[1].sqlID} } case 74: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:477 + //line sql.y:478 { yyVAL.smTableExpr = &TableName{Qualifier: yyDollar[1].sqlID, Name: yyDollar[3].sqlID} } case 75: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:481 + //line sql.y:482 { yyVAL.smTableExpr = yyDollar[1].subquery } case 76: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:487 + //line sql.y:488 { yyVAL.tableName = &TableName{Name: yyDollar[1].sqlID} } case 77: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:491 + //line sql.y:492 { yyVAL.tableName = &TableName{Qualifier: yyDollar[1].sqlID, Name: yyDollar[3].sqlID} } case 78: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:496 + //line sql.y:497 { yyVAL.indexHints = nil } case 79: yyDollar = yyS[yypt-5 : yypt+1] - //line sql.y:500 + //line sql.y:501 { yyVAL.indexHints = &IndexHints{Type: AST_USE, Indexes: yyDollar[4].sqlIDs} } case 80: yyDollar = yyS[yypt-5 : yypt+1] - //line sql.y:504 + //line sql.y:505 { yyVAL.indexHints = &IndexHints{Type: AST_IGNORE, Indexes: yyDollar[4].sqlIDs} } case 81: yyDollar = yyS[yypt-5 : yypt+1] - //line sql.y:508 + //line sql.y:509 { yyVAL.indexHints = &IndexHints{Type: AST_FORCE, Indexes: yyDollar[4].sqlIDs} } case 82: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:514 + //line sql.y:515 { yyVAL.sqlIDs = []SQLName{yyDollar[1].sqlID} } case 83: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:518 + //line sql.y:519 { yyVAL.sqlIDs = append(yyDollar[1].sqlIDs, yyDollar[3].sqlID) } case 84: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:523 + //line sql.y:524 { yyVAL.boolExpr = nil } case 85: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:527 + //line sql.y:528 { yyVAL.boolExpr = yyDollar[2].boolExpr } case 87: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:534 + //line sql.y:535 { yyVAL.boolExpr = &AndExpr{Left: yyDollar[1].boolExpr, Right: yyDollar[3].boolExpr} } case 88: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:538 + //line sql.y:539 { yyVAL.boolExpr = &OrExpr{Left: yyDollar[1].boolExpr, Right: yyDollar[3].boolExpr} } case 89: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:542 + //line sql.y:543 { yyVAL.boolExpr = &NotExpr{Expr: yyDollar[2].boolExpr} } case 90: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:546 + //line sql.y:547 { yyVAL.boolExpr = &ParenBoolExpr{Expr: yyDollar[2].boolExpr} } case 91: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:552 + //line sql.y:553 { yyVAL.boolExpr = &ComparisonExpr{Left: yyDollar[1].valExpr, Operator: yyDollar[2].str, Right: yyDollar[3].valExpr} } case 92: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:556 + //line sql.y:557 { yyVAL.boolExpr = &ComparisonExpr{Left: yyDollar[1].valExpr, Operator: AST_IN, Right: yyDollar[3].colTuple} } case 93: yyDollar = yyS[yypt-4 : yypt+1] - //line sql.y:560 + //line sql.y:561 { yyVAL.boolExpr = &ComparisonExpr{Left: yyDollar[1].valExpr, Operator: AST_NOT_IN, Right: yyDollar[4].colTuple} } case 94: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:564 + //line sql.y:565 { yyVAL.boolExpr = &ComparisonExpr{Left: yyDollar[1].valExpr, Operator: AST_LIKE, Right: yyDollar[3].valExpr} } case 95: yyDollar = yyS[yypt-4 : yypt+1] - //line sql.y:568 + //line sql.y:569 { yyVAL.boolExpr = &ComparisonExpr{Left: yyDollar[1].valExpr, Operator: AST_NOT_LIKE, Right: yyDollar[4].valExpr} } case 96: yyDollar = yyS[yypt-5 : yypt+1] - //line sql.y:572 + //line sql.y:573 { yyVAL.boolExpr = &RangeCond{Left: yyDollar[1].valExpr, Operator: AST_BETWEEN, From: yyDollar[3].valExpr, To: yyDollar[5].valExpr} } case 97: yyDollar = yyS[yypt-6 : yypt+1] - //line sql.y:576 + //line sql.y:577 { yyVAL.boolExpr = &RangeCond{Left: yyDollar[1].valExpr, Operator: AST_NOT_BETWEEN, From: yyDollar[4].valExpr, To: yyDollar[6].valExpr} } case 98: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:580 + //line sql.y:581 { yyVAL.boolExpr = &NullCheck{Operator: AST_IS_NULL, Expr: yyDollar[1].valExpr} } case 99: yyDollar = yyS[yypt-4 : yypt+1] - //line sql.y:584 + //line sql.y:585 { yyVAL.boolExpr = &NullCheck{Operator: AST_IS_NOT_NULL, Expr: yyDollar[1].valExpr} } case 100: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:588 + //line sql.y:589 { yyVAL.boolExpr = &ExistsExpr{Subquery: yyDollar[2].subquery} } case 101: yyDollar = yyS[yypt-6 : yypt+1] - //line sql.y:592 + //line sql.y:593 { yyVAL.boolExpr = &KeyrangeExpr{Start: yyDollar[3].valExpr, End: yyDollar[5].valExpr} } case 102: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:598 + //line sql.y:599 { yyVAL.str = AST_EQ } case 103: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:602 + //line sql.y:603 { yyVAL.str = AST_LT } case 104: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:606 + //line sql.y:607 { yyVAL.str = AST_GT } case 105: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:610 + //line sql.y:611 { yyVAL.str = AST_LE } case 106: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:614 + //line sql.y:615 { yyVAL.str = AST_GE } case 107: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:618 + //line sql.y:619 { yyVAL.str = AST_NE } case 108: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:622 + //line sql.y:623 { yyVAL.str = AST_NSE } case 109: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:628 + //line sql.y:629 { yyVAL.colTuple = ValTuple(yyDollar[2].valExprs) } case 110: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:632 + //line sql.y:633 { yyVAL.colTuple = yyDollar[1].subquery } case 111: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:636 + //line sql.y:637 { yyVAL.colTuple = ListArg(yyDollar[1].bytes) } case 112: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:642 + //line sql.y:643 { yyVAL.subquery = &Subquery{yyDollar[2].selStmt} } case 113: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:648 + //line sql.y:649 { yyVAL.valExprs = ValExprs{yyDollar[1].valExpr} } case 114: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:652 + //line sql.y:653 { yyVAL.valExprs = append(yyDollar[1].valExprs, yyDollar[3].valExpr) } case 115: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:658 + //line sql.y:659 { yyVAL.valExpr = yyDollar[1].valExpr } case 116: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:662 + //line sql.y:663 { yyVAL.valExpr = yyDollar[1].colName } case 117: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:666 + //line sql.y:667 { yyVAL.valExpr = yyDollar[1].rowTuple } case 118: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:670 + //line sql.y:671 { yyVAL.valExpr = &BinaryExpr{Left: yyDollar[1].valExpr, Operator: AST_BITAND, Right: yyDollar[3].valExpr} } case 119: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:674 + //line sql.y:675 { yyVAL.valExpr = &BinaryExpr{Left: yyDollar[1].valExpr, Operator: AST_BITOR, Right: yyDollar[3].valExpr} } case 120: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:678 + //line sql.y:679 { yyVAL.valExpr = &BinaryExpr{Left: yyDollar[1].valExpr, Operator: AST_BITXOR, Right: yyDollar[3].valExpr} } case 121: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:682 + //line sql.y:683 { yyVAL.valExpr = &BinaryExpr{Left: yyDollar[1].valExpr, Operator: AST_PLUS, Right: yyDollar[3].valExpr} } case 122: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:686 + //line sql.y:687 { yyVAL.valExpr = &BinaryExpr{Left: yyDollar[1].valExpr, Operator: AST_MINUS, Right: yyDollar[3].valExpr} } case 123: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:690 + //line sql.y:691 { yyVAL.valExpr = &BinaryExpr{Left: yyDollar[1].valExpr, Operator: AST_MULT, Right: yyDollar[3].valExpr} } case 124: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:694 + //line sql.y:695 { yyVAL.valExpr = &BinaryExpr{Left: yyDollar[1].valExpr, Operator: AST_DIV, Right: yyDollar[3].valExpr} } case 125: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:698 + //line sql.y:699 { yyVAL.valExpr = &BinaryExpr{Left: yyDollar[1].valExpr, Operator: AST_MOD, Right: yyDollar[3].valExpr} } case 126: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:702 + //line sql.y:703 { yyVAL.valExpr = &BinaryExpr{Left: yyDollar[1].valExpr, Operator: AST_SHIFT_LEFT, Right: yyDollar[3].valExpr} } case 127: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:706 + //line sql.y:707 { yyVAL.valExpr = &BinaryExpr{Left: yyDollar[1].valExpr, Operator: AST_SHIFT_RIGHT, Right: yyDollar[3].valExpr} } case 128: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:710 + //line sql.y:711 { if num, ok := yyDollar[2].valExpr.(NumVal); ok { yyVAL.valExpr = num @@ -1631,7 +1639,7 @@ yydefault: } case 129: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:718 + //line sql.y:719 { if num, ok := yyDollar[2].valExpr.(NumVal); ok { // Handle double negative @@ -1646,241 +1654,241 @@ yydefault: } case 130: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:731 + //line sql.y:732 { yyVAL.valExpr = &UnaryExpr{Operator: AST_TILDA, Expr: yyDollar[2].valExpr} } case 131: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:735 + //line sql.y:736 { yyVAL.valExpr = &FuncExpr{Name: string(yyDollar[1].sqlID)} } case 132: yyDollar = yyS[yypt-4 : yypt+1] - //line sql.y:739 + //line sql.y:740 { yyVAL.valExpr = &FuncExpr{Name: string(yyDollar[1].sqlID), Exprs: yyDollar[3].selectExprs} } case 133: yyDollar = yyS[yypt-5 : yypt+1] - //line sql.y:743 + //line sql.y:744 { yyVAL.valExpr = &FuncExpr{Name: string(yyDollar[1].sqlID), Distinct: true, Exprs: yyDollar[4].selectExprs} } case 134: yyDollar = yyS[yypt-4 : yypt+1] - //line sql.y:747 + //line sql.y:748 { yyVAL.valExpr = &FuncExpr{Name: yyDollar[1].str, Exprs: yyDollar[3].selectExprs} } case 135: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:751 + //line sql.y:752 { yyVAL.valExpr = yyDollar[1].caseExpr } case 136: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:757 + //line sql.y:758 { yyVAL.str = "if" } case 137: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:761 + //line sql.y:762 { yyVAL.str = "values" } case 138: yyDollar = yyS[yypt-5 : yypt+1] - //line sql.y:767 + //line sql.y:768 { yyVAL.caseExpr = &CaseExpr{Expr: yyDollar[2].valExpr, Whens: yyDollar[3].whens, Else: yyDollar[4].valExpr} } case 139: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:772 + //line sql.y:773 { yyVAL.valExpr = nil } case 140: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:776 + //line sql.y:777 { yyVAL.valExpr = yyDollar[1].valExpr } case 141: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:782 + //line sql.y:783 { yyVAL.whens = []*When{yyDollar[1].when} } case 142: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:786 + //line sql.y:787 { yyVAL.whens = append(yyDollar[1].whens, yyDollar[2].when) } case 143: yyDollar = yyS[yypt-4 : yypt+1] - //line sql.y:792 + //line sql.y:793 { yyVAL.when = &When{Cond: yyDollar[2].boolExpr, Val: yyDollar[4].valExpr} } case 144: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:797 + //line sql.y:798 { yyVAL.valExpr = nil } case 145: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:801 + //line sql.y:802 { yyVAL.valExpr = yyDollar[2].valExpr } case 146: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:807 + //line sql.y:808 { yyVAL.colName = &ColName{Name: yyDollar[1].sqlID} } case 147: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:811 + //line sql.y:812 { yyVAL.colName = &ColName{Qualifier: yyDollar[1].sqlID, Name: yyDollar[3].sqlID} } case 148: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:817 + //line sql.y:818 { yyVAL.valExpr = StrVal(yyDollar[1].bytes) } case 149: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:821 + //line sql.y:822 { yyVAL.valExpr = NumVal(yyDollar[1].bytes) } case 150: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:825 + //line sql.y:826 { yyVAL.valExpr = ValArg(yyDollar[1].bytes) } case 151: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:829 + //line sql.y:830 { yyVAL.valExpr = &NullVal{} } case 152: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:834 + //line sql.y:835 { yyVAL.valExprs = nil } case 153: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:838 + //line sql.y:839 { yyVAL.valExprs = yyDollar[3].valExprs } case 154: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:843 + //line sql.y:844 { yyVAL.boolExpr = nil } case 155: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:847 + //line sql.y:848 { yyVAL.boolExpr = yyDollar[2].boolExpr } case 156: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:852 + //line sql.y:853 { yyVAL.orderBy = nil } case 157: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:856 + //line sql.y:857 { yyVAL.orderBy = yyDollar[3].orderBy } case 158: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:862 + //line sql.y:863 { yyVAL.orderBy = OrderBy{yyDollar[1].order} } case 159: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:866 + //line sql.y:867 { yyVAL.orderBy = append(yyDollar[1].orderBy, yyDollar[3].order) } case 160: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:872 + //line sql.y:873 { yyVAL.order = &Order{Expr: yyDollar[1].valExpr, Direction: yyDollar[2].str} } case 161: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:877 + //line sql.y:878 { yyVAL.str = AST_ASC } case 162: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:881 + //line sql.y:882 { yyVAL.str = AST_ASC } case 163: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:885 + //line sql.y:886 { yyVAL.str = AST_DESC } case 164: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:890 + //line sql.y:891 { yyVAL.limit = nil } case 165: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:894 + //line sql.y:895 { yyVAL.limit = &Limit{Rowcount: yyDollar[2].valExpr} } case 166: yyDollar = yyS[yypt-4 : yypt+1] - //line sql.y:898 + //line sql.y:899 { yyVAL.limit = &Limit{Offset: yyDollar[2].valExpr, Rowcount: yyDollar[4].valExpr} } case 167: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:903 + //line sql.y:904 { yyVAL.str = "" } case 168: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:907 + //line sql.y:908 { yyVAL.str = AST_FOR_UPDATE } case 169: yyDollar = yyS[yypt-4 : yypt+1] - //line sql.y:911 + //line sql.y:912 { if yyDollar[3].sqlID != "share" { yylex.Error("expecting share") @@ -1894,211 +1902,211 @@ yydefault: } case 170: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:924 + //line sql.y:925 { yyVAL.columns = nil } case 171: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:928 + //line sql.y:929 { yyVAL.columns = yyDollar[2].columns } case 172: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:934 + //line sql.y:935 { yyVAL.columns = Columns{&NonStarExpr{Expr: yyDollar[1].colName}} } case 173: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:938 + //line sql.y:939 { yyVAL.columns = append(yyVAL.columns, &NonStarExpr{Expr: yyDollar[3].colName}) } case 174: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:943 + //line sql.y:944 { yyVAL.updateExprs = nil } case 175: yyDollar = yyS[yypt-5 : yypt+1] - //line sql.y:947 + //line sql.y:948 { yyVAL.updateExprs = yyDollar[5].updateExprs } case 176: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:953 + //line sql.y:954 { yyVAL.insRows = yyDollar[2].values } case 177: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:957 + //line sql.y:958 { yyVAL.insRows = yyDollar[1].selStmt } case 178: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:963 + //line sql.y:964 { yyVAL.values = Values{yyDollar[1].rowTuple} } case 179: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:967 + //line sql.y:968 { yyVAL.values = append(yyDollar[1].values, yyDollar[3].rowTuple) } case 180: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:973 + //line sql.y:974 { yyVAL.rowTuple = ValTuple(yyDollar[2].valExprs) } case 181: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:977 + //line sql.y:978 { yyVAL.rowTuple = yyDollar[1].subquery } case 182: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:983 + //line sql.y:984 { yyVAL.updateExprs = UpdateExprs{yyDollar[1].updateExpr} } case 183: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:987 + //line sql.y:988 { yyVAL.updateExprs = append(yyDollar[1].updateExprs, yyDollar[3].updateExpr) } case 184: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:993 + //line sql.y:994 { yyVAL.updateExpr = &UpdateExpr{Name: yyDollar[1].colName, Expr: yyDollar[3].valExpr} } case 185: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:998 + //line sql.y:999 { yyVAL.empty = struct{}{} } case 186: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:1000 + //line sql.y:1001 { yyVAL.empty = struct{}{} } case 187: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:1003 + //line sql.y:1004 { yyVAL.empty = struct{}{} } case 188: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:1005 + //line sql.y:1006 { yyVAL.empty = struct{}{} } case 189: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:1008 + //line sql.y:1009 { yyVAL.empty = struct{}{} } case 190: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:1010 + //line sql.y:1011 { yyVAL.empty = struct{}{} } case 191: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:1014 + //line sql.y:1015 { yyVAL.empty = struct{}{} } case 192: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:1016 + //line sql.y:1017 { yyVAL.empty = struct{}{} } case 193: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:1018 + //line sql.y:1019 { yyVAL.empty = struct{}{} } case 194: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:1020 + //line sql.y:1021 { yyVAL.empty = struct{}{} } case 195: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:1022 + //line sql.y:1023 { yyVAL.empty = struct{}{} } case 196: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:1025 + //line sql.y:1026 { yyVAL.empty = struct{}{} } case 197: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:1027 + //line sql.y:1028 { yyVAL.empty = struct{}{} } case 198: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:1030 + //line sql.y:1031 { yyVAL.empty = struct{}{} } case 199: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:1032 + //line sql.y:1033 { yyVAL.empty = struct{}{} } case 200: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:1035 + //line sql.y:1036 { yyVAL.empty = struct{}{} } case 201: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:1037 + //line sql.y:1038 { yyVAL.empty = struct{}{} } case 202: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:1041 + //line sql.y:1042 { yyVAL.sqlID = SQLName(strings.ToLower(string(yyDollar[1].bytes))) } case 203: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:1047 + //line sql.y:1048 { yyVAL.sqlID = SQLName(yyDollar[1].bytes) } case 204: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:1053 + //line sql.y:1054 { if incNesting(yylex) { yylex.Error("max nesting level reached") @@ -2107,13 +2115,13 @@ yydefault: } case 205: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:1062 + //line sql.y:1063 { decNesting(yylex) } case 206: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:1067 + //line sql.y:1068 { forceEOF(yylex) } diff --git a/go/vt/sqlparser/sql.y b/go/vt/sqlparser/sql.y index ba937a8a0b..500afb5e2e 100644 --- a/go/vt/sqlparser/sql.y +++ b/go/vt/sqlparser/sql.y @@ -74,6 +74,7 @@ func forceEOF(yylex interface{}) { %token LEX_ERROR %token SELECT INSERT UPDATE DELETE FROM WHERE GROUP HAVING ORDER BY LIMIT FOR %token ALL DISTINCT AS EXISTS IN IS LIKE BETWEEN NULL ASC DESC VALUES INTO DUPLICATE KEY DEFAULT SET LOCK KEYRANGE +%token VALUE, LAST_INSERT_ID %token ID STRING NUMBER VALUE_ARG LIST_ARG COMMENT %token LE GE NE NULL_SAFE_EQUAL %token '(' '=' '<' '>' diff --git a/go/vt/sqlparser/token.go b/go/vt/sqlparser/token.go index 05fb3d1878..808da42599 100644 --- a/go/vt/sqlparser/token.go +++ b/go/vt/sqlparser/token.go @@ -36,76 +36,78 @@ func NewStringTokenizer(sql string) *Tokenizer { } var keywords = map[string]int{ - "all": ALL, - "alter": ALTER, - "analyze": ANALYZE, - "and": AND, - "as": AS, - "asc": ASC, - "between": BETWEEN, - "by": BY, - "case": CASE, - "create": CREATE, - "cross": CROSS, - "default": DEFAULT, - "delete": DELETE, - "desc": DESC, - "describe": DESCRIBE, - "distinct": DISTINCT, - "drop": DROP, - "duplicate": DUPLICATE, - "else": ELSE, - "end": END, - "except": EXCEPT, - "exists": EXISTS, - "explain": EXPLAIN, - "for": FOR, - "force": FORCE, - "from": FROM, - "group": GROUP, - "having": HAVING, - "if": IF, - "ignore": IGNORE, - "in": IN, - "index": INDEX, - "inner": INNER, - "insert": INSERT, - "intersect": INTERSECT, - "into": INTO, - "is": IS, - "join": JOIN, - "key": KEY, - "keyrange": KEYRANGE, - "left": LEFT, - "like": LIKE, - "limit": LIMIT, - "lock": LOCK, - "minus": MINUS, - "natural": NATURAL, - "not": NOT, - "null": NULL, - "on": ON, - "or": OR, - "order": ORDER, - "outer": OUTER, - "rename": RENAME, - "right": RIGHT, - "select": SELECT, - "set": SET, - "show": SHOW, - "straight_join": STRAIGHT_JOIN, - "table": TABLE, - "then": THEN, - "to": TO, - "union": UNION, - "unique": UNIQUE, - "update": UPDATE, - "use": USE, - "using": USING, - "values": VALUES, - "view": VIEW, - "when": WHEN, - "where": WHERE, + "all": ALL, + "alter": ALTER, + "analyze": ANALYZE, + "and": AND, + "as": AS, + "asc": ASC, + "between": BETWEEN, + "by": BY, + "case": CASE, + "create": CREATE, + "cross": CROSS, + "default": DEFAULT, + "delete": DELETE, + "desc": DESC, + "describe": DESCRIBE, + "distinct": DISTINCT, + "drop": DROP, + "duplicate": DUPLICATE, + "else": ELSE, + "end": END, + "except": EXCEPT, + "exists": EXISTS, + "explain": EXPLAIN, + "for": FOR, + "force": FORCE, + "from": FROM, + "group": GROUP, + "having": HAVING, + "if": IF, + "ignore": IGNORE, + "in": IN, + "index": INDEX, + "inner": INNER, + "insert": INSERT, + "intersect": INTERSECT, + "into": INTO, + "is": IS, + "join": JOIN, + "key": KEY, + "keyrange": KEYRANGE, + "last_insert_id": LAST_INSERT_ID, + "left": LEFT, + "like": LIKE, + "limit": LIMIT, + "lock": LOCK, + "minus": MINUS, + "natural": NATURAL, + "not": NOT, + "null": NULL, + "on": ON, + "or": OR, + "order": ORDER, + "outer": OUTER, + "rename": RENAME, + "right": RIGHT, + "select": SELECT, + "set": SET, + "show": SHOW, + "straight_join": STRAIGHT_JOIN, + "table": TABLE, + "then": THEN, + "to": TO, + "union": UNION, + "unique": UNIQUE, + "update": UPDATE, + "use": USE, + "using": USING, + "values": VALUES, + "value": VALUE, + "view": VIEW, + "when": WHEN, + "where": WHERE, } // Lex returns the next token form the Tokenizer. From d3fc76999f981e48df387166c618c77b80641b8a Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Fri, 14 Aug 2015 13:23:33 -0700 Subject: [PATCH 02/80] tabletserver: upsert WIP Upsert is mostly feature complete. There are still two issues to resolve: 1. RowsAffected is returned as 0 if no rows were modified. So, we cannot assume that no rows were matched. So, the value cannot be used to verify if the dup key matched a pk or a unique key. 2. I mistooke VALUES to be VALUE, and coded defense against using that construct in the parser. It turns out that the parser already allows use of VALUES. So, I'll have to find a different way to prevent that usage for upserts. --- data/test/tabletserver/exec_cases.txt | 180 ++++++++++++++++++-- data/test/tabletserver/stream_cases.txt | 3 + go/vt/tabletserver/dbconn_test.go | 2 +- go/vt/tabletserver/planbuilder/dml.go | 47 +++-- go/vt/tabletserver/planbuilder/plan.go | 13 +- go/vt/tabletserver/planbuilder/query_gen.go | 12 +- go/vt/tabletserver/query_executor.go | 42 ++++- go/vt/tabletserver/query_executor_test.go | 56 ++++++ go/vt/tabletserver/sqlquery_test.go | 10 +- go/vt/tabletserver/table_info_test.go | 5 +- go/vt/tabletserver/tx_pool_test.go | 6 +- go/vt/vttest/fakesqldb/conn.go | 33 ++-- test/queryservice_tests/nocache_cases.py | 40 +++++ test/queryservice_tests/nocache_tests.py | 25 ++- test/test_data/table_acl_config.json | 2 +- test/test_data/test_schema.sql | 4 + 16 files changed, 406 insertions(+), 74 deletions(-) diff --git a/data/test/tabletserver/exec_cases.txt b/data/test/tabletserver/exec_cases.txt index c53ef2c95a..d757953fcd 100644 --- a/data/test/tabletserver/exec_cases.txt +++ b/data/test/tabletserver/exec_cases.txt @@ -8,6 +8,7 @@ "FullQuery": "select * from a union select * from b", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": null, @@ -28,6 +29,7 @@ "FullQuery": "select distinct * from a limit :#maxLimit", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": null, @@ -48,6 +50,7 @@ "FullQuery": "select * from a group by b limit :#maxLimit", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": null, @@ -68,6 +71,7 @@ "FullQuery": "select * from a having b = 1 limit :#maxLimit", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": null, @@ -88,6 +92,7 @@ "FullQuery": "select * from a limit 5", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": [ 0, @@ -113,6 +118,7 @@ "FullQuery": "select * from a.b limit :#maxLimit", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": null, @@ -133,6 +139,7 @@ "FullQuery": "select * from a, b limit :#maxLimit", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": null, @@ -153,6 +160,7 @@ "FullQuery": "select * from a join b limit :#maxLimit", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": null, @@ -173,6 +181,7 @@ "FullQuery": "select * from a right join b limit :#maxLimit", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": null, @@ -193,6 +202,7 @@ "FullQuery": "select * from b limit :#maxLimit", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": null, @@ -213,6 +223,7 @@ "FullQuery": "select * from (b) limit :#maxLimit", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": null, @@ -233,6 +244,7 @@ "FullQuery": "select :bv from a limit :#maxLimit", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": null, @@ -253,6 +265,7 @@ "FullQuery": "select eid + 1 from a limit :#maxLimit", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": null, @@ -273,6 +286,7 @@ "FullQuery": "select case when eid = 1 then 1 end from a limit :#maxLimit", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": null, @@ -293,6 +307,7 @@ "FullQuery": "select eid from a limit :#maxLimit", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": [ 0 @@ -315,6 +330,7 @@ "FullQuery": "select eid as foo from a limit :#maxLimit", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": [ 0 @@ -337,6 +353,7 @@ "FullQuery": "select * from a limit :#maxLimit", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": [ 0, @@ -362,6 +379,7 @@ "FullQuery": "select c.eid from a as c limit :#maxLimit", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": [ 0 @@ -384,6 +402,7 @@ "FullQuery": "select (eid) from a limit :#maxLimit", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": null, @@ -404,6 +423,7 @@ "FullQuery": "select eid from a limit :#maxLimit for update", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": null, @@ -424,6 +444,7 @@ "FullQuery": "select eid from a limit :#maxLimit lock in share mode", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": null, @@ -444,6 +465,7 @@ "FullQuery": "select * from a where eid = 1 and id in (1, 2) limit :#maxLimit", "OuterQuery": "select eid, id, name, foo from a where :#pk", "Subquery": null, + "UpsertQuery": null, "IndexUsed": "PRIMARY", "ColumnNumbers": [ 0, @@ -469,6 +491,7 @@ "FullQuery": "select * from a where eid = :v1 and id in (:v2, :v3) limit :#maxLimit", "OuterQuery": "select eid, id, name, foo from a where :#pk", "Subquery": null, + "UpsertQuery": null, "IndexUsed": "PRIMARY", "ColumnNumbers": [ 0, @@ -494,6 +517,7 @@ "FullQuery": "select * from a where name = 'foo' limit :#maxLimit", "OuterQuery": "select eid, id, name, foo from a where :#pk", "Subquery": "select eid, id from a use index (b_name) where name = 'foo' limit :#maxLimit", + "UpsertQuery": null, "IndexUsed": "b_name", "ColumnNumbers": [ 0, @@ -519,6 +543,7 @@ "FullQuery": "select eid, name, id from a where name = 'foo' limit :#maxLimit", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "b_name", "ColumnNumbers": [ 0, @@ -543,6 +568,7 @@ "FullQuery": "select * from d where id = 1 limit :#maxLimit", "OuterQuery": "select name, id, foo, bar from d where :#pk", "Subquery": "select name from d use index (d_id) where id = 1 limit :#maxLimit", + "UpsertQuery": null, "IndexUsed": "d_id", "ColumnNumbers": [ 0, @@ -568,6 +594,7 @@ "FullQuery": "select * from d where id = 1 limit 1", "OuterQuery": "select name, id, foo, bar from d where :#pk", "Subquery": "select name from d use index (d_id) where id = 1 limit 1", + "UpsertQuery": null, "IndexUsed": "d_id", "ColumnNumbers": [ 0, @@ -593,6 +620,7 @@ "FullQuery": "select * from a where eid + 1 = 1 limit :#maxLimit", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": [ 0, @@ -618,6 +646,7 @@ "FullQuery": "select * from a where eid = id limit :#maxLimit", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": [ 0, @@ -643,6 +672,7 @@ "FullQuery": "select * from d where name between 'foo' and 'bar' limit :#maxLimit", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "PRIMARY", "ColumnNumbers": [ 0, @@ -668,6 +698,7 @@ "FullQuery": "select * from a where (eid = 1) and (id = 2) limit :#maxLimit", "OuterQuery": "select eid, id, name, foo from a where :#pk", "Subquery": null, + "UpsertQuery": null, "IndexUsed": "PRIMARY", "ColumnNumbers": [ 0, @@ -696,6 +727,7 @@ "FullQuery": "select * from a where eid = 1 and id = 1 limit :#maxLimit", "OuterQuery": "select eid, id, name, foo from a where :#pk", "Subquery": null, + "UpsertQuery": null, "IndexUsed": "PRIMARY", "ColumnNumbers": [ 0, @@ -724,6 +756,7 @@ "FullQuery": "select * from d where bar = 'foo' and id = 1 limit :#maxLimit", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": [ 0, @@ -749,6 +782,7 @@ "FullQuery": "select * from d where name = 'foo' limit :#maxLimit", "OuterQuery": "select name, id, foo, bar from d where :#pk", "Subquery": null, + "UpsertQuery": null, "IndexUsed": "PRIMARY", "ColumnNumbers": [ 0, @@ -776,6 +810,7 @@ "FullQuery": "select * from d where name = 'foo' limit 1", "OuterQuery": "select name, id, foo, bar from d where :#pk", "Subquery": null, + "UpsertQuery": null, "IndexUsed": "PRIMARY", "ColumnNumbers": [ 0, @@ -803,6 +838,7 @@ "FullQuery": "select * from d where name = 'foo' limit 0", "OuterQuery": "select name, id, foo, bar from d where :#pk", "Subquery": null, + "UpsertQuery": null, "IndexUsed": "PRIMARY", "ColumnNumbers": [ 0, @@ -830,6 +866,7 @@ "FullQuery": "select * from d where name = 'foo' limit :a", "OuterQuery": "select name, id, foo, bar from d where :#pk", "Subquery": null, + "UpsertQuery": null, "IndexUsed": "PRIMARY", "ColumnNumbers": [ 0, @@ -857,6 +894,7 @@ "FullQuery": "select * from d where name = 'foo' limit 1, 1", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "PRIMARY", "ColumnNumbers": [ 0, @@ -894,6 +932,7 @@ "FullQuery": "select * from d where 'foo' = name and eid = 1 limit :#maxLimit", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": [ 0, @@ -919,6 +958,7 @@ "FullQuery": "select * from d where name in ('foo', 'bar') limit :#maxLimit", "OuterQuery": "select name, id, foo, bar from d where :#pk", "Subquery": null, + "UpsertQuery": null, "IndexUsed": "PRIMARY", "ColumnNumbers": [ 0, @@ -947,6 +987,7 @@ "FullQuery": "select * from d where name in (:a, :b) limit :#maxLimit", "OuterQuery": "select name, id, foo, bar from d where :#pk", "Subquery": null, + "UpsertQuery": null, "IndexUsed": "PRIMARY", "ColumnNumbers": [ 0, @@ -975,6 +1016,7 @@ "FullQuery": "select * from d where name in ('foo') limit :#maxLimit", "OuterQuery": "select name, id, foo, bar from d where :#pk", "Subquery": null, + "UpsertQuery": null, "IndexUsed": "PRIMARY", "ColumnNumbers": [ 0, @@ -1002,6 +1044,7 @@ "FullQuery": "select * from d where name in (:a) limit :#maxLimit", "OuterQuery": "select name, id, foo, bar from d where :#pk", "Subquery": null, + "UpsertQuery": null, "IndexUsed": "PRIMARY", "ColumnNumbers": [ 0, @@ -1029,6 +1072,7 @@ "FullQuery": "select * from d where name in (:a) limit 1", "OuterQuery": "select name, id, foo, bar from d where :#pk", "Subquery": null, + "UpsertQuery": null, "IndexUsed": "PRIMARY", "ColumnNumbers": [ 0, @@ -1054,6 +1098,7 @@ "FullQuery": "select * from a where eid in (1) and id in (1, 2) limit :#maxLimit", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": [ 0, @@ -1079,6 +1124,7 @@ "FullQuery": "select * from a where eid in (1, 2) and id in (1, 2) limit :#maxLimit", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": [ 0, @@ -1104,6 +1150,7 @@ "FullQuery": "select * from a where (eid, id) in ((1, 1), (2, 2)) limit :#maxLimit", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": [ 0, @@ -1129,6 +1176,7 @@ "FullQuery":"select * from d where foo = 'bar' limit :#maxLimit", "OuterQuery": null, "Subquery":null, + "UpsertQuery": null, "IndexUsed":"", "ColumnNumbers": [ 0, @@ -1154,6 +1202,7 @@ "FullQuery": "select * from d as c where c.name = 'foo' limit :#maxLimit", "OuterQuery": "select name, id, foo, bar from d as c where :#pk", "Subquery": null, + "UpsertQuery": null, "IndexUsed": "PRIMARY", "ColumnNumbers": [ 0, @@ -1181,6 +1230,7 @@ "FullQuery": "select * from d where id \u003c 0 limit :#maxLimit", "OuterQuery": "select name, id, foo, bar from d where :#pk", "Subquery": "select name from d use index (d_id) where id \u003c 0 limit :#maxLimit", + "UpsertQuery": null, "IndexUsed": "d_id", "ColumnNumbers": [ 0, @@ -1206,6 +1256,7 @@ "FullQuery": "select * from d where name in ('foo', id) limit :#maxLimit", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": [ 0, @@ -1231,6 +1282,7 @@ "FullQuery": "select * from d where id between 1 and 2 limit :#maxLimit", "OuterQuery": "select name, id, foo, bar from d where :#pk", "Subquery": "select name from d use index (d_id) where id between 1 and 2 limit :#maxLimit", + "UpsertQuery": null, "IndexUsed": "d_id", "ColumnNumbers": [ 0, @@ -1256,6 +1308,7 @@ "FullQuery": "select * from d where id not between 1 and 2 limit :#maxLimit", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": [ 0, @@ -1281,6 +1334,7 @@ "FullQuery": "select * from d where 1 between 1 and 2 limit :#maxLimit", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": [ 0, @@ -1306,6 +1360,7 @@ "FullQuery": "select * from d where name is not null limit :#maxLimit", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": [ 0, @@ -1331,6 +1386,7 @@ "FullQuery": "select * from a where eid = 1 and id = 1 order by name asc limit :#maxLimit", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": [ 0, @@ -1356,6 +1412,7 @@ "FullQuery": "select * from d where bar = 'foo' limit :#maxLimit", "OuterQuery": "select name, id, foo, bar from d where :#pk", "Subquery": "select name from d use index (d_bar) where bar = 'foo' limit :#maxLimit", + "UpsertQuery": null, "IndexUsed": "d_bar", "ColumnNumbers": [ 0, @@ -1381,6 +1438,7 @@ "FullQuery": "select * from d use index (d_bar_never) where bar = 'foo' limit :#maxLimit", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": [ 0, @@ -1406,6 +1464,7 @@ "FullQuery": "select * from d force index (d_bar_never) where bar = 'foo' limit :#maxLimit", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": [ 0, @@ -1435,6 +1494,7 @@ "FullQuery": "insert into b.a(eid, id) values (1, :a)", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": null, @@ -1455,6 +1515,7 @@ "FullQuery": "insert into a(a.eid, id) values (1, 2)", "OuterQuery": "insert into a(a.eid, id) values (1, 2)", "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": [ @@ -1482,6 +1543,7 @@ "FullQuery": "insert into a(eid, id) values (1, :a)", "OuterQuery": "insert into a(eid, id) values (1, :a)", "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": [ @@ -1505,6 +1567,7 @@ "FullQuery": "insert into a(id) values (1)", "OuterQuery": "insert into a(id) values (1)", "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": [ @@ -1528,6 +1591,7 @@ "FullQuery": "insert into d(id) values (1)", "OuterQuery": "insert into d(id) values (1)", "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": [ @@ -1554,6 +1618,7 @@ "FullQuery": "insert into a(eid, id) values (-1, 2)", "OuterQuery": "insert into a(eid, id) values (-1, 2)", "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": [ @@ -1577,6 +1642,7 @@ "FullQuery": "insert into a(eid, id) values (1, 2)", "OuterQuery": "insert into a(eid, id) values (1, 2)", "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": [ @@ -1594,12 +1660,13 @@ "insert into a (eid, id) values (~1, 2)" { "PlanId": "PASS_DML", - "Reason": "DEFAULT", + "Reason": "COMPLEX_EXPR", "TableName": "a", "FieldQuery": null, "FullQuery": "insert into a(eid, id) values (~1, 2)", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": null, @@ -1614,12 +1681,13 @@ "insert into a (eid, id) values (1+1, 2)" { "PlanId": "PASS_DML", - "Reason": "DEFAULT", + "Reason": "COMPLEX_EXPR", "TableName": "a", "FieldQuery": null, "FullQuery": "insert into a(eid, id) values (1 + 1, 2)", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": null, @@ -1640,6 +1708,7 @@ "FullQuery": "insert into c(eid, id) values (1, 2)", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": null, @@ -1660,6 +1729,7 @@ "FullQuery": "insert into a values (1, 2)", "OuterQuery": "insert into a values (1, 2)", "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": [ @@ -1674,18 +1744,19 @@ } # on dup -"insert into b (eid, id) values (1, 2) on duplicate key update name = values(a)" +"insert into b (eid, id) values (1, 2) on duplicate key update name = func(a)" { - "PlanId": "PASS_DML", - "Reason": "UPSERT", + "PlanId": "UPSERT_PK", + "Reason": "DEFAULT", "TableName": "b", "FieldQuery": null, - "FullQuery": "insert into b(eid, id) values (1, 2) on duplicate key update name = values(a)", - "OuterQuery": null, + "FullQuery": "insert into b(eid, id) values (1, 2) on duplicate key update name = func(a)", + "OuterQuery": "insert into b(eid, id) values (1, 2)", "Subquery": null, + "UpsertQuery": "update b set name = func(a) where :#pk", "IndexUsed": "", "ColumnNumbers": null, - "PKValues": null, + "PKValues": [1, 2], "Limit": null, "SecondaryPKValues": null, "SubqueryPKColumns": null, @@ -1696,16 +1767,38 @@ # on dup pk change "insert into b (eid, id) values (1, 2) on duplicate key update eid = 2" { - "PlanId": "PASS_DML", - "Reason": "UPSERT", + "PlanId": "UPSERT_PK", + "Reason": "DEFAULT", "TableName": "b", "FieldQuery": null, "FullQuery": "insert into b(eid, id) values (1, 2) on duplicate key update eid = 2", - "OuterQuery": null, + "OuterQuery": "insert into b(eid, id) values (1, 2)", "Subquery": null, + "UpsertQuery": "update b set eid = 2 where :#pk", "IndexUsed": "", "ColumnNumbers": null, - "PKValues": null, + "PKValues": [1, 2], + "Limit": null, + "SecondaryPKValues": [2, null], + "SubqueryPKColumns": null, + "SetKey": "", + "SetValue": null +} + +# on dup complex pk change +"insert into b (id, eid) values (1, 2) on duplicate key update eid = func(a)" +{ + "PlanId": "PASS_DML", + "Reason": "PK_CHANGE", + "TableName": "b", + "FieldQuery": null, + "FullQuery": "insert into b(id, eid) values (1, 2) on duplicate key update eid = func(a)", + "OuterQuery": "insert into b(id, eid) values (1, 2)", + "Subquery": null, + "UpsertQuery": null, + "IndexUsed": "", + "ColumnNumbers": null, + "PKValues": [2, 1], "Limit": null, "SecondaryPKValues": null, "SubqueryPKColumns": null, @@ -1713,16 +1806,38 @@ "SetValue": null } -# on dup complex pk change -"insert into b (id, eid) values (1, 2) on duplicate key update eid = values(a)" +# on dup multi-row +"insert into b (id, eid) values (1, 2), (2, 3) on duplicate key update name = func(a)" { "PlanId": "PASS_DML", "Reason": "UPSERT", "TableName": "b", "FieldQuery": null, - "FullQuery": "insert into b(id, eid) values (1, 2) on duplicate key update eid = values(a)", + "FullQuery": "insert into b(id, eid) values (1, 2), (2, 3) on duplicate key update name = func(a)", + "OuterQuery": "insert into b(id, eid) values (1, 2), (2, 3)", + "Subquery": null, + "UpsertQuery": null, + "IndexUsed": "", + "ColumnNumbers": null, + "PKValues": [[2,3],[1,2]], + "Limit": null, + "SecondaryPKValues": null, + "SubqueryPKColumns": null, + "SetKey": "", + "SetValue": null +} + +# on dup subquery +"insert into b (id, eid) select * from a on duplicate key update name = func(a)" +{ + "PlanId": "PASS_DML", + "Reason": "UPSERT", + "TableName": "b", + "FieldQuery": null, + "FullQuery": "insert into b(id, eid) select * from a on duplicate key update name = func(a)", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": null, @@ -1743,6 +1858,7 @@ "FullQuery": "insert into b(eid, id) select * from a", "OuterQuery": "insert into b(eid, id) values :#values", "Subquery": "select * from a limit :#maxLimit", + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": [ 0, @@ -1769,6 +1885,7 @@ "FullQuery": "insert into b select * from a", "OuterQuery": "insert into b values :#values", "Subquery": "select * from a limit :#maxLimit", + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": [ 0, @@ -1795,6 +1912,7 @@ "FullQuery": "insert into b(eid, id) values (1, 2), (3, 4)", "OuterQuery": "insert into b(eid, id) values (1, 2), (3, 4)", "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": [ @@ -1824,6 +1942,7 @@ "FullQuery": "update b.a set name = 'foo' where eid = 1 and id = 1", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": null, @@ -1844,6 +1963,7 @@ "FullQuery": "update b set eid = 1", "OuterQuery": "update b set eid = 1 where :#pk", "Subquery": "select eid, id from b limit :#maxLimit for update", + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": null, @@ -1871,6 +1991,7 @@ "FullQuery": "update b set a.eid = 1", "OuterQuery": "update b set a.eid = 1 where :#pk", "Subquery": "select eid, id from b limit :#maxLimit for update", + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": null, @@ -1894,6 +2015,7 @@ "FullQuery": "update b set eid = foo()", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": null, @@ -1914,6 +2036,7 @@ "FullQuery": "update a set name = 'foo'", "OuterQuery": "update a set name = 'foo' where :#pk", "Subquery": "select eid, id from a limit :#maxLimit for update", + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": null, @@ -1934,6 +2057,7 @@ "FullQuery": "update a set name = 'foo' where eid + 1 = 1", "OuterQuery": "update a set name = 'foo' where :#pk", "Subquery": "select eid, id from a where eid + 1 = 1 limit :#maxLimit for update", + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": null, @@ -1954,6 +2078,7 @@ "FullQuery": "update a set name = 'foo' where eid = 1 and id = 1", "OuterQuery": "update a set name = 'foo' where :#pk", "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": [ @@ -1977,6 +2102,7 @@ "FullQuery": "update a set a.name = 'foo' where eid = 1 and id = 1", "OuterQuery": "update a set a.name = 'foo' where :#pk", "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": [ @@ -2000,6 +2126,7 @@ "FullQuery": "update a set name = 'foo' where eid = 1", "OuterQuery": "update a set name = 'foo' where :#pk", "Subquery": "select eid, id from a where eid = 1 limit :#maxLimit for update", + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": null, @@ -2020,6 +2147,7 @@ "FullQuery": "update a set name = 'foo' where eid = 1 limit 10", "OuterQuery": "update a set name = 'foo' where :#pk", "Subquery": "select eid, id from a where eid = 1 limit 10 for update", + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": null, @@ -2040,6 +2168,7 @@ "FullQuery": "update a set name = 'foo' where eid = 1 and name = 'foo'", "OuterQuery": "update a set name = 'foo' where :#pk", "Subquery": "select eid, id from a where eid = 1 and name = 'foo' limit :#maxLimit for update", + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": null, @@ -2060,6 +2189,7 @@ "FullQuery": "update c set eid = 1", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": null, @@ -2080,6 +2210,7 @@ "FullQuery": "delete from b.a where eid = 1 and id = 1", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": null, @@ -2100,6 +2231,7 @@ "FullQuery": "delete from a", "OuterQuery": "delete from a where :#pk", "Subquery": "select eid, id from a limit :#maxLimit for update", + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": null, @@ -2120,6 +2252,7 @@ "FullQuery": "delete from a where eid + 1 = 1", "OuterQuery": "delete from a where :#pk", "Subquery": "select eid, id from a where eid + 1 = 1 limit :#maxLimit for update", + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": null, @@ -2140,6 +2273,7 @@ "FullQuery": "delete from a where eid = 1 and id = 1", "OuterQuery": "delete from a where :#pk", "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": [ @@ -2163,6 +2297,7 @@ "FullQuery": "delete from a where eid = 1", "OuterQuery": "delete from a where :#pk", "Subquery": "select eid, id from a where eid = 1 limit :#maxLimit for update", + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": null, @@ -2183,6 +2318,7 @@ "FullQuery": "delete from a where eid = 1 and name = 'foo'", "OuterQuery": "delete from a where :#pk", "Subquery": "select eid, id from a where eid = 1 and name = 'foo' limit :#maxLimit for update", + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": null, @@ -2203,6 +2339,7 @@ "FullQuery": "delete from c", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": null, @@ -2223,6 +2360,7 @@ "FullQuery": "set a = 1", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": null, @@ -2243,6 +2381,7 @@ "FullQuery": "set a = 1.2", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": null, @@ -2263,6 +2402,7 @@ "FullQuery": "set a = 'b'", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": null, @@ -2283,6 +2423,7 @@ "FullQuery": "set a = 1, b = 2", "OuterQuery": null, "Subquery": null, + "UpsertQuery": null, "IndexUsed": "", "ColumnNumbers": null, "PKValues": null, @@ -2303,6 +2444,7 @@ "FullQuery":null, "OuterQuery":null, "Subquery":null, + "UpsertQuery": null, "IndexUsed":"", "ColumnNumbers":null, "PKValues":null, @@ -2323,6 +2465,7 @@ "FullQuery":null, "OuterQuery":null, "Subquery":null, + "UpsertQuery": null, "IndexUsed":"", "ColumnNumbers":null, "PKValues":null, @@ -2343,6 +2486,7 @@ "FullQuery":null, "OuterQuery":null, "Subquery":null, + "UpsertQuery": null, "IndexUsed":"", "ColumnNumbers":null, "PKValues":null, @@ -2363,6 +2507,7 @@ "FullQuery":null, "OuterQuery":null, "Subquery":null, + "UpsertQuery": null, "IndexUsed":"", "ColumnNumbers":null, "PKValues":null, @@ -2383,6 +2528,7 @@ "FullQuery":null, "OuterQuery":null, "Subquery":null, + "UpsertQuery": null, "IndexUsed":"", "ColumnNumbers":null, "PKValues":null, @@ -2403,6 +2549,7 @@ "FullQuery":null, "OuterQuery":null, "Subquery":null, + "UpsertQuery": null, "IndexUsed":"", "ColumnNumbers":null, "PKValues":null, @@ -2423,6 +2570,7 @@ "FullQuery":null, "OuterQuery":null, "Subquery":null, + "UpsertQuery": null, "IndexUsed":"", "ColumnNumbers":null, "PKValues":null, @@ -2443,6 +2591,7 @@ "FullQuery":null, "OuterQuery":null, "Subquery":null, + "UpsertQuery": null, "IndexUsed":"", "ColumnNumbers":null, "PKValues":null, @@ -2463,6 +2612,7 @@ "FullQuery":null, "OuterQuery":null, "Subquery":null, + "UpsertQuery": null, "IndexUsed":"", "ColumnNumbers":null, "PKValues":null, diff --git a/data/test/tabletserver/stream_cases.txt b/data/test/tabletserver/stream_cases.txt index ba3ecdd8fe..d61dacd9ef 100644 --- a/data/test/tabletserver/stream_cases.txt +++ b/data/test/tabletserver/stream_cases.txt @@ -8,6 +8,7 @@ "FullQuery":"select * from a", "OuterQuery":null, "Subquery":null, + "UpsertQuery":null, "IndexUsed":"", "ColumnNumbers":null, "PKValues":null, @@ -28,6 +29,7 @@ "FullQuery":"select * from a join b", "OuterQuery":null, "Subquery":null, + "UpsertQuery":null, "IndexUsed":"", "ColumnNumbers":null, "PKValues":null, @@ -52,6 +54,7 @@ "FullQuery": "select * from a union select * from b", "OuterQuery":null, "Subquery":null, + "UpsertQuery":null, "IndexUsed":"", "ColumnNumbers":null, "PKValues":null, diff --git a/go/vt/tabletserver/dbconn_test.go b/go/vt/tabletserver/dbconn_test.go index f031fafbe0..bae8749802 100644 --- a/go/vt/tabletserver/dbconn_test.go +++ b/go/vt/tabletserver/dbconn_test.go @@ -84,7 +84,7 @@ func TestDBConnKill(t *testing.T) { } newKillQuery := fmt.Sprintf("kill %d", dbConn.ID()) // Kill failed because "kill query_id" failed - db.AddRejectedQuery(newKillQuery) + db.AddRejectedQuery(newKillQuery, errRejected) err = dbConn.Kill() testUtils.checkTabletError(t, err, ErrFail, "Could not kill query") diff --git a/go/vt/tabletserver/planbuilder/dml.go b/go/vt/tabletserver/planbuilder/dml.go index 81c3209d9d..8a8ef808c3 100644 --- a/go/vt/tabletserver/planbuilder/dml.go +++ b/go/vt/tabletserver/planbuilder/dml.go @@ -388,14 +388,13 @@ func analyzeInsert(ins *sqlparser.Insert, getTable TableGetter) (plan *ExecPlan, pkColumnNumbers := getInsertPKColumns(ins.Columns, tableInfo) - if ins.OnDup != nil { - // Upserts are not safe for statement based replication: - // http://bugs.mysql.com/bug.php?id=58637 - plan.Reason = REASON_UPSERT - return plan, nil - } - if sel, ok := ins.Rows.(sqlparser.SelectStatement); ok { + if ins.OnDup != nil { + // Upserts not allowed for subqueries. + // http://bugs.mysql.com/bug.php?id=58637 + plan.Reason = REASON_UPSERT + return plan, nil + } plan.PlanId = PLAN_INSERT_SUBQUERY plan.OuterQuery = GenerateInsertOuterQuery(ins) plan.Subquery = GenerateSelectLimitQuery(sel) @@ -422,11 +421,36 @@ func analyzeInsert(ins *sqlparser.Insert, getTable TableGetter) (plan *ExecPlan, if err != nil { return nil, err } - if pkValues != nil { - plan.PlanId = PLAN_INSERT_PK - plan.OuterQuery = plan.FullQuery - plan.PKValues = pkValues + if pkValues == nil { + plan.Reason = REASON_COMPLEX_EXPR + return plan, nil } + plan.OuterQuery = GenerateInsertNoUpdate(ins) + plan.PKValues = pkValues + if ins.OnDup == nil { + plan.PlanId = PLAN_INSERT_PK + return plan, nil + } + if len(rowList) > 1 { + // Upsert supported only for single row inserts. + plan.Reason = REASON_UPSERT + return plan, nil + } + plan.SecondaryPKValues, err = analyzeUpdateExpressions(sqlparser.UpdateExprs(ins.OnDup), tableInfo.Indexes[0]) + if err != nil { + if err == ErrTooComplex { + plan.Reason = REASON_PK_CHANGE + return plan, nil + } + return nil, err + } + plan.PlanId = PLAN_UPSERT_PK + upd := &sqlparser.Update{ + Comments: ins.Comments, + Table: ins.Table, + Exprs: sqlparser.UpdateExprs(ins.OnDup), + } + plan.UpsertQuery = GenerateUpdateOuterQuery(upd) return plan, nil } @@ -467,7 +491,6 @@ func getInsertPKValues(pkColumnNumbers []int, rowList sqlparser.Values, tableInf } node := row[columnNumber] if !sqlparser.IsValue(node) { - log.Warningf("insert is too complex %v", node) return nil, nil } var err error diff --git a/go/vt/tabletserver/planbuilder/plan.go b/go/vt/tabletserver/planbuilder/plan.go index 288e390e7f..e88505d465 100644 --- a/go/vt/tabletserver/planbuilder/plan.go +++ b/go/vt/tabletserver/planbuilder/plan.go @@ -54,6 +54,8 @@ const ( PLAN_SELECT_STREAM // PLAN_OTHER is for SHOW, DESCRIBE & EXPLAIN statements PLAN_OTHER + // PLAN_UPSERT_PK is for insert ... on duplicate key constructs + PLAN_UPSERT_PK // NumPlans stores the total number of plans NumPlans ) @@ -73,6 +75,7 @@ var planName = []string{ "DDL", "SELECT_STREAM", "OTHER", + "UPSERT_PK", } func (pt PlanType) String() string { @@ -120,6 +123,7 @@ var tableAclRoles = map[PlanType]tableacl.Role{ PLAN_DDL: tableacl.ADMIN, PLAN_SELECT_STREAM: tableacl.READER, PLAN_OTHER: tableacl.ADMIN, + PLAN_UPSERT_PK: tableacl.WRITER, } // ReasonType indicates why a query plan fails to build @@ -141,6 +145,7 @@ const ( REASON_TABLE_NOINDEX REASON_PK_CHANGE REASON_HAS_HINTS + REASON_COMPLEX_EXPR REASON_UPSERT ) @@ -161,6 +166,7 @@ var reasonName = []string{ "TABLE_NOINDEX", "PK_CHANGE", "HAS_HINTS", + "COMPLEX_EXPR", "UPSERT", } @@ -193,9 +199,10 @@ type ExecPlan struct { // For PK plans, only OuterQuery is set. // For SUBQUERY plans, Subquery is also set. // IndexUsed is set only for PLAN_SELECT_SUBQUERY - OuterQuery *sqlparser.ParsedQuery - Subquery *sqlparser.ParsedQuery - IndexUsed string + OuterQuery *sqlparser.ParsedQuery + Subquery *sqlparser.ParsedQuery + UpsertQuery *sqlparser.ParsedQuery + IndexUsed string // For selects, columns to be returned // For PLAN_INSERT_SUBQUERY, columns to be inserted diff --git a/go/vt/tabletserver/planbuilder/query_gen.go b/go/vt/tabletserver/planbuilder/query_gen.go index 4724e5be92..b96144e74a 100644 --- a/go/vt/tabletserver/planbuilder/query_gen.go +++ b/go/vt/tabletserver/planbuilder/query_gen.go @@ -78,16 +78,24 @@ func GenerateSelectOuterQuery(sel *sqlparser.Select, tableInfo *schema.Table) *s // GenerateInsertOuterQuery generates the outer query for inserts. func GenerateInsertOuterQuery(ins *sqlparser.Insert) *sqlparser.ParsedQuery { buf := sqlparser.NewTrackedBuffer(nil) - buf.Myprintf("insert %vinto %v%v values %a%v", + buf.Myprintf("insert %vinto %v%v values %a", ins.Comments, ins.Table, ins.Columns, ":#values", - ins.OnDup, ) return buf.ParsedQuery() } +// GenerateInsertNoUpdate generates an insert without the update part. +func GenerateInsertNoUpdate(ins *sqlparser.Insert) *sqlparser.ParsedQuery { + buf := sqlparser.NewTrackedBuffer(nil) + buf.Myprintf("insert %vinto %v%v %v", + ins.Comments, + ins.Table, ins.Columns, ins.Rows) + return buf.ParsedQuery() +} + // GenerateUpdateOuterQuery generates the outer query for updates. func GenerateUpdateOuterQuery(upd *sqlparser.Update) *sqlparser.ParsedQuery { buf := sqlparser.NewTrackedBuffer(nil) diff --git a/go/vt/tabletserver/query_executor.go b/go/vt/tabletserver/query_executor.go index f7153c4f5e..1a0290c20d 100644 --- a/go/vt/tabletserver/query_executor.go +++ b/go/vt/tabletserver/query_executor.go @@ -10,6 +10,7 @@ import ( log "github.com/golang/glog" "github.com/youtube/vitess/go/hack" + "github.com/youtube/vitess/go/mysql" mproto "github.com/youtube/vitess/go/mysql/proto" "github.com/youtube/vitess/go/sqltypes" "github.com/youtube/vitess/go/vt/callinfo" @@ -88,6 +89,8 @@ func (qre *QueryExecutor) Execute() (reply *mproto.QueryResult, err error) { reply, err = qre.execDMLSubquery(conn, invalidator) case planbuilder.PLAN_OTHER: reply, err = qre.execSQL(conn, qre.query, true) + case planbuilder.PLAN_UPSERT_PK: + reply, err = qre.execUpsertPK(conn, invalidator) default: // select or set in a transaction, just count as select reply, err = qre.execDirect(conn) } @@ -180,6 +183,8 @@ func (qre *QueryExecutor) execDmlAutoCommit() (reply *mproto.QueryResult, err er reply, err = qre.execDMLPK(conn, invalidator) case planbuilder.PLAN_DML_SUBQUERY: reply, err = qre.execDMLSubquery(conn, invalidator) + case planbuilder.PLAN_UPSERT_PK: + reply, err = qre.execUpsertPK(conn, invalidator) default: return nil, NewTabletError(ErrFatal, "unsupported query: %s", qre.query) } @@ -466,12 +471,35 @@ func (qre *QueryExecutor) execInsertSubquery(conn poolConn) (*mproto.QueryResult } func (qre *QueryExecutor) execInsertPKRows(conn poolConn, pkRows [][]sqltypes.Value) (*mproto.QueryResult, error) { - secondaryList, err := buildSecondaryList(qre.plan.TableInfo, pkRows, qre.plan.SecondaryPKValues, qre.bindVars) + bsc := buildStreamComment(qre.plan.TableInfo, pkRows, nil) + return qre.directFetch(conn, qre.plan.OuterQuery, qre.bindVars, bsc) +} + +func (qre *QueryExecutor) execUpsertPK(conn poolConn, invalidator CacheInvalidator) (*mproto.QueryResult, error) { + pkRows, err := buildValueList(qre.plan.TableInfo, qre.plan.PKValues, qre.bindVars) if err != nil { return nil, err } - bsc := buildStreamComment(qre.plan.TableInfo, pkRows, secondaryList) - return qre.directFetch(conn, qre.plan.OuterQuery, qre.bindVars, bsc) + bsc := buildStreamComment(qre.plan.TableInfo, pkRows, nil) + result, err := qre.directFetch(conn, qre.plan.OuterQuery, qre.bindVars, bsc) + if err == nil { + return result, nil + } + terr, ok := err.(*TabletError) + if !ok { + return result, err + } + if terr.SqlError != mysql.ErrDupEntry { + return nil, err + } + result, err = qre.execDMLPKRows(conn, qre.plan.UpsertQuery, pkRows, invalidator) + if err != nil { + return nil, err + } + if result.RowsAffected != 1 { + return nil, NewTabletError(ErrFail, "upsert failed to update a dup key row") + } + return result, nil } func (qre *QueryExecutor) execDMLPK(conn poolConn, invalidator CacheInvalidator) (*mproto.QueryResult, error) { @@ -479,7 +507,7 @@ func (qre *QueryExecutor) execDMLPK(conn poolConn, invalidator CacheInvalidator) if err != nil { return nil, err } - return qre.execDMLPKRows(conn, pkRows, invalidator) + return qre.execDMLPKRows(conn, qre.plan.OuterQuery, pkRows, invalidator) } func (qre *QueryExecutor) execDMLSubquery(conn poolConn, invalidator CacheInvalidator) (*mproto.QueryResult, error) { @@ -487,10 +515,10 @@ func (qre *QueryExecutor) execDMLSubquery(conn poolConn, invalidator CacheInvali if err != nil { return nil, err } - return qre.execDMLPKRows(conn, innerResult.Rows, invalidator) + return qre.execDMLPKRows(conn, qre.plan.OuterQuery, innerResult.Rows, invalidator) } -func (qre *QueryExecutor) execDMLPKRows(conn poolConn, pkRows [][]sqltypes.Value, invalidator CacheInvalidator) (*mproto.QueryResult, error) { +func (qre *QueryExecutor) execDMLPKRows(conn poolConn, query *sqlparser.ParsedQuery, pkRows [][]sqltypes.Value, invalidator CacheInvalidator) (*mproto.QueryResult, error) { if len(pkRows) == 0 { return &mproto.QueryResult{RowsAffected: 0}, nil } @@ -516,7 +544,7 @@ func (qre *QueryExecutor) execDMLPKRows(conn poolConn, pkRows [][]sqltypes.Value Columns: qre.plan.TableInfo.Indexes[0].Columns, Rows: pkRows, } - r, err := qre.directFetch(conn, qre.plan.OuterQuery, qre.bindVars, bsc) + r, err := qre.directFetch(conn, query, qre.bindVars, bsc) if err != nil { return nil, err } diff --git a/go/vt/tabletserver/query_executor_test.go b/go/vt/tabletserver/query_executor_test.go index 38ca0a6193..bcd0fa3db3 100644 --- a/go/vt/tabletserver/query_executor_test.go +++ b/go/vt/tabletserver/query_executor_test.go @@ -12,7 +12,9 @@ import ( "testing" "time" + "github.com/youtube/vitess/go/mysql" mproto "github.com/youtube/vitess/go/mysql/proto" + "github.com/youtube/vitess/go/sqldb" "github.com/youtube/vitess/go/sqltypes" "github.com/youtube/vitess/go/vt/callinfo" tableaclpb "github.com/youtube/vitess/go/vt/proto/tableacl" @@ -214,6 +216,60 @@ func TestQueryExecutorPlanInsertSubQuery(t *testing.T) { } } +func TestQueryExecutorPlanUpsertPk(t *testing.T) { + db := setUpQueryExecutorTest() + db.AddQuery("insert into test_table values (1) /* _stream test_table (pk ) (1 ); */", &mproto.QueryResult{}) + want := &mproto.QueryResult{ + Rows: make([][]sqltypes.Value, 0), + } + query := "insert into test_table values(1) on duplicate key update val=1" + ctx := context.Background() + sqlQuery := newTestSQLQuery(ctx, enableRowCache|enableStrict) + qre := newTestQueryExecutor(ctx, sqlQuery, query, 0) + defer sqlQuery.disallowQueries() + checkPlanID(t, planbuilder.PLAN_UPSERT_PK, qre.plan.PlanId) + got, err := qre.Execute() + if err != nil { + t.Fatalf("qre.Execute() = %v, want nil", err) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("got: %v, want: %v", got, want) + } + + db.AddRejectedQuery("insert into test_table values (1) /* _stream test_table (pk ) (1 ); */", errRejected) + _, err = qre.Execute() + wantErr := "error: rejected" + if err == nil || err.Error() != wantErr { + t.Fatalf("qre.Execute() = %v, want %v", err, wantErr) + } + + db.AddRejectedQuery( + "insert into test_table values (1) /* _stream test_table (pk ) (1 ); */", + sqldb.NewSqlError(mysql.ErrDupEntry, "err"), + ) + db.AddQuery("update test_table set val = 1 where pk in (1) /* _stream test_table (pk ) (1 ); */", &mproto.QueryResult{}) + _, err = qre.Execute() + wantErr = "error: upsert failed to update a dup key row" + if err == nil || err.Error() != wantErr { + t.Fatalf("qre.Execute() = %v, want %v", err, wantErr) + } + + db.AddQuery( + "update test_table set val = 1 where pk in (1) /* _stream test_table (pk ) (1 ); */", + &mproto.QueryResult{RowsAffected: 1}, + ) + got, err = qre.Execute() + if err != nil { + t.Fatalf("qre.Execute() = %v, want nil", err) + } + want = &mproto.QueryResult{ + RowsAffected: 1, + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("got: %v, want: %v", got, want) + } +} + func TestQueryExecutorPlanDmlPk(t *testing.T) { db := setUpQueryExecutorTest() query := "update test_table set name = 2 where pk in (1) /* _stream test_table (pk ) (1 ); */" diff --git a/go/vt/tabletserver/sqlquery_test.go b/go/vt/tabletserver/sqlquery_test.go index 26485f9b39..15201079be 100644 --- a/go/vt/tabletserver/sqlquery_test.go +++ b/go/vt/tabletserver/sqlquery_test.go @@ -527,7 +527,7 @@ func TestSqlQueryExecuteBatchBeginFail(t *testing.T) { db := setUpSqlQueryTest() testUtils := newTestUtils() // make "begin" query fail - db.AddRejectedQuery("begin") + db.AddRejectedQuery("begin", errRejected) config := testUtils.newQueryServiceConfig() sqlQuery := NewSqlQuery(config) dbconfigs := testUtils.newDBConfigs() @@ -561,7 +561,7 @@ func TestSqlQueryExecuteBatchCommitFail(t *testing.T) { db := setUpSqlQueryTest() testUtils := newTestUtils() // make "commit" query fail - db.AddRejectedQuery("commit") + db.AddRejectedQuery("commit", errRejected) config := testUtils.newQueryServiceConfig() sqlQuery := NewSqlQuery(config) dbconfigs := testUtils.newDBConfigs() @@ -607,8 +607,8 @@ func TestSqlQueryExecuteBatchSqlExecFailInTransaction(t *testing.T) { db.AddQuery(expanedSql, sqlResult) // make this query fail - db.AddRejectedQuery(sql) - db.AddRejectedQuery(expanedSql) + db.AddRejectedQuery(sql, errRejected) + db.AddRejectedQuery(expanedSql, errRejected) config := testUtils.newQueryServiceConfig() sqlQuery := NewSqlQuery(config) @@ -662,7 +662,7 @@ func TestSqlQueryExecuteBatchSqlSucceedInTransaction(t *testing.T) { db.AddQuery(expanedSql, sqlResult) // cause execution error for this particular sql query - db.AddRejectedQuery(sql) + db.AddRejectedQuery(sql, errRejected) config := testUtils.newQueryServiceConfig() config.EnableAutoCommit = true diff --git a/go/vt/tabletserver/table_info_test.go b/go/vt/tabletserver/table_info_test.go index 77ee6d3c43..424e7e850e 100644 --- a/go/vt/tabletserver/table_info_test.go +++ b/go/vt/tabletserver/table_info_test.go @@ -5,6 +5,7 @@ package tabletserver import ( + "errors" "fmt" "math/rand" "testing" @@ -18,6 +19,8 @@ import ( "golang.org/x/net/context" ) +var errRejected = errors.New("rejected") + func TestTableInfoNew(t *testing.T) { fakecacheservice.Register() db := fakesqldb.Register() @@ -46,7 +49,7 @@ func TestTableInfoFailBecauseUnableToRetrieveTableIndex(t *testing.T) { for query, result := range getTestTableInfoQueries() { db.AddQuery(query, result) } - db.AddRejectedQuery("show index from `test_table`") + db.AddRejectedQuery("show index from `test_table`", errRejected) cachePool := newTestTableInfoCachePool() cachePool.Open() defer cachePool.Close() diff --git a/go/vt/tabletserver/tx_pool_test.go b/go/vt/tabletserver/tx_pool_test.go index d59d415a18..a51b77d647 100644 --- a/go/vt/tabletserver/tx_pool_test.go +++ b/go/vt/tabletserver/tx_pool_test.go @@ -174,7 +174,7 @@ func TestTxPoolBeginWithPoolConnectionError(t *testing.T) { func TestTxPoolBeginWithExecError(t *testing.T) { db := fakesqldb.Register() - db.AddRejectedQuery("begin") + db.AddRejectedQuery("begin", errRejected) txPool := newTxPool(false) appParams := sqldb.ConnParams{} dbaParams := sqldb.ConnParams{} @@ -190,7 +190,7 @@ func TestTxPoolSafeCommitFail(t *testing.T) { sql := fmt.Sprintf("alter table test_table add test_column int") db.AddQuery("begin", &proto.QueryResult{}) db.AddQuery(sql, &proto.QueryResult{}) - db.AddRejectedQuery("commit") + db.AddRejectedQuery("commit", errRejected) txPool := newTxPool(false) appParams := sqldb.ConnParams{} dbaParams := sqldb.ConnParams{} @@ -215,7 +215,7 @@ func TestTxPoolRollbackFail(t *testing.T) { db := fakesqldb.Register() db.AddQuery(sql, &proto.QueryResult{}) db.AddQuery("begin", &proto.QueryResult{}) - db.AddRejectedQuery("rollback") + db.AddRejectedQuery("rollback", errRejected) txPool := newTxPool(false) appParams := sqldb.ConnParams{} diff --git a/go/vt/vttest/fakesqldb/conn.go b/go/vt/vttest/fakesqldb/conn.go index fde7a1272d..5a3abb4ff3 100644 --- a/go/vt/vttest/fakesqldb/conn.go +++ b/go/vt/vttest/fakesqldb/conn.go @@ -31,7 +31,7 @@ type Conn struct { type DB struct { isConnFail bool data map[string]*proto.QueryResult - rejectedData map[string]*proto.QueryResult + rejectedData map[string]error queryCalled map[string]int mu sync.Mutex } @@ -67,18 +67,10 @@ func (db *DB) DeleteQuery(query string) { } // AddRejectedQuery adds a query which will be rejected at execution time. -func (db *DB) AddRejectedQuery(query string) { +func (db *DB) AddRejectedQuery(query string, err error) { db.mu.Lock() defer db.mu.Unlock() - db.rejectedData[strings.ToLower(query)] = &proto.QueryResult{} -} - -// HasRejectedQuery returns true if this query will be rejected. -func (db *DB) HasRejectedQuery(query string) bool { - db.mu.Lock() - defer db.mu.Unlock() - _, ok := db.rejectedData[strings.ToLower(query)] - return ok + db.rejectedData[strings.ToLower(query)] = err } // DeleteRejectedQuery deletes query from the fake DB. @@ -138,8 +130,8 @@ func (conn *Conn) ExecuteFetch(query string, maxrows int, wantfields bool) (*pro if conn.IsClosed() { return nil, fmt.Errorf("connection is closed") } - if conn.db.HasRejectedQuery(query) { - return nil, fmt.Errorf("unsupported query, reject query: %s", query) + if err, ok := conn.db.rejectedData[query]; ok { + return nil, err } result, ok := conn.db.GetQuery(query) if !ok { @@ -158,12 +150,9 @@ func (conn *Conn) ExecuteFetch(query string, maxrows int, wantfields bool) (*pro copy(qr.Fields, result.Fields) } - rowCount := int(qr.RowsAffected) - rows := make([][]sqltypes.Value, rowCount) - if rowCount > 0 { - for i := 0; i < rowCount; i++ { - rows[i] = result.Rows[i] - } + rows := make([][]sqltypes.Value, 0, len(result.Rows)) + for _, r := range result.Rows { + rows = append(rows, r) } qr.Rows = rows return qr, nil @@ -204,8 +193,8 @@ func (conn *Conn) ExecuteStreamFetch(query string) error { if conn.IsClosed() { return fmt.Errorf("connection is closed") } - if conn.db.HasRejectedQuery(query) { - return fmt.Errorf("unsupported query, reject query: %s", query) + if err, ok := conn.db.rejectedData[query]; ok { + return err } result, ok := conn.db.GetQuery(query) if !ok { @@ -283,7 +272,7 @@ func Register() *DB { name := fmt.Sprintf("fake-%d", rand.Int63()) db := &DB{ data: make(map[string]*proto.QueryResult), - rejectedData: make(map[string]*proto.QueryResult), + rejectedData: make(map[string]error), queryCalled: make(map[string]int), } sqldb.Register(name, func(sqldb.ConnParams) (sqldb.Conn, error) { diff --git a/test/queryservice_tests/nocache_cases.py b/test/queryservice_tests/nocache_cases.py index 218dc88abf..16d27c6949 100644 --- a/test/queryservice_tests/nocache_cases.py +++ b/test/queryservice_tests/nocache_cases.py @@ -411,6 +411,46 @@ cases = [ 'delete from vtocc_a where eid>1', 'commit']), + MultiCase( + 'upsert single row present/absent', + ['begin', + Case(sql="insert into upsert_test(id1, id2) values (1, 1) on duplicate key update id2 = 1", + rewritten="insert into upsert_test(id1, id2) values (1, 1) /* _stream upsert_test (id1 ) (1 )", + rowcount=1), + Case(sql='select * from upsert_test', + result=[(1L, 1L)]), + Case(sql="insert into upsert_test(id1, id2) values (1, 2) on duplicate key update id2 = 2", + rewritten=[ + "insert into upsert_test(id1, id2) values (1, 2) /* _stream upsert_test (id1 ) (1 )", + "update upsert_test set id2 = 2 where id1 in (1) /* _stream upsert_test (id1 ) (1 )"], + rowcount=1), + Case(sql='select * from upsert_test', + result=[(1L, 2L)]), + 'commit', + 'begin', + 'delete from upsert_test', + 'commit']), + + MultiCase( + 'upsert changes pk', + ['begin', + Case(sql="insert into upsert_test(id1, id2) values (1, 1) on duplicate key update id1 = 1", + rewritten="insert into upsert_test(id1, id2) values (1, 1) /* _stream upsert_test (id1 ) (1 )", + rowcount=1), + Case(sql='select * from upsert_test', + result=[(1L, 1L)]), + Case(sql="insert into upsert_test(id1, id2) values (1, 2) on duplicate key update id1 = 2", + rewritten=[ + "insert into upsert_test(id1, id2) values (1, 2) /* _stream upsert_test (id1 ) (1 )", + "update upsert_test set id1 = 2 where id1 in (1) /* _stream upsert_test (id1 ) (1 ) (2 )"], + rowcount=1), + Case(sql='select * from upsert_test', + result=[(2L, 1L)]), + 'commit', + 'begin', + 'delete from upsert_test', + 'commit']), + MultiCase( 'update', ['begin', diff --git a/test/queryservice_tests/nocache_tests.py b/test/queryservice_tests/nocache_tests.py index 48f6f89d6b..26cd6bbca4 100644 --- a/test/queryservice_tests/nocache_tests.py +++ b/test/queryservice_tests/nocache_tests.py @@ -142,12 +142,33 @@ class TestNocache(framework.TestCase): self.env.execute("insert into vtocc_a(eid, id, name, foo) values (7, 1+1, '', '')") with self.assertRaises(dbexceptions.DatabaseError): self.env.execute("insert into vtocc_d(eid, id) values (1, 1)") - with self.assertRaises(dbexceptions.DatabaseError): - self.env.execute("insert into vtocc_a(eid, id, name, foo) values (8, 2, '', '') on duplicate key update id = 2+1") with self.assertRaises(dbexceptions.DatabaseError): self.env.execute("update vtocc_a set eid = 1+1 where eid = 1 and id = 1") with self.assertRaises(dbexceptions.DatabaseError): self.env.execute("insert into vtocc_d(eid, id) values (1, 1)") + + self.env.execute("delete from upsert_test") + with self.assertRaises(dbexceptions.DatabaseError): + self.env.execute("insert into upsert_test(id1, id2) values (1, 1), (2, 2) on duplicate key update id1 = 1") + + self.env.execute("delete from upsert_test") + with self.assertRaises(dbexceptions.DatabaseError): + self.env.execute("insert into upsert_test(id1, id2) select eid, id from vtocc_a limit 1 on duplicate key update id2 = id1") + + self.env.execute("delete from upsert_test") + with self.assertRaises(dbexceptions.DatabaseError): + self.env.execute("insert into upsert_test(id1, id2) values (1, 1) on duplicate key update id1 = 2+1") + + self.env.execute("delete from upsert_test") + with self.assertRaises(dbexceptions.DatabaseError): + self.env.execute("insert into upsert_test(id1, id2) values (1, 1)") + self.env.execute("insert into upsert_test(id1, id2) values (2, 1) on duplicate key update id2 = 2") + + # TODO(sougou): add test for values clause after bug fix on handling + # values. + self.env.execute("delete from upsert_test") + with self.assertRaises(dbexceptions.DatabaseError): + self.env.execute("insert into upsert_test(id1, id2) values (1, 1) on duplicate key update id2 = last_insert_id(id1)") finally: self.env.conn.rollback() diff --git a/test/test_data/table_acl_config.json b/test/test_data/table_acl_config.json index 9c5d1cbd54..b73195a5d3 100644 --- a/test/test_data/table_acl_config.json +++ b/test/test_data/table_acl_config.json @@ -30,7 +30,7 @@ }, { "name": "vtocc", - "table_names_or_prefixes": ["vtocc_a", "vtocc_b", "vtocc_c", "dual", "vtocc_d", "vtocc_temp", "vtocc_e", "vtocc_f", "vtocc_strings", "vtocc_fracts", "vtocc_ints", "vtocc_misc", "vtocc_big", "vtocc_view"], + "table_names_or_prefixes": ["vtocc_a", "vtocc_b", "vtocc_c", "dual", "vtocc_d", "vtocc_temp", "vtocc_e", "vtocc_f", "upsert_test", "vtocc_strings", "vtocc_fracts", "vtocc_ints", "vtocc_misc", "vtocc_big", "vtocc_view"], "readers": ["youtube-dev-dedicated"], "writers": ["youtube-dev-dedicated"], "admins": ["youtube-dev-dedicated"] diff --git a/test/test_data/test_schema.sql b/test/test_data/test_schema.sql index b4caf3d1a5..c3aa2971a7 100644 --- a/test/test_data/test_schema.sql +++ b/test/test_data/test_schema.sql @@ -12,12 +12,15 @@ create table vtocc_c(eid bigint, name varchar(128), foo varbinary(128), primary create table vtocc_d(eid bigint, id int) comment 'vtocc_nocache' create table vtocc_e(eid bigint auto_increment, id int default 1, name varchar(128) default 'name', foo varchar(128), primary key(eid, id, name)) comment 'vtocc_nocache' create table vtocc_f(vb varbinary(16) default 'ab', id int, primary key(vb)) comment 'vtocc_nocache' +create table upsert_test(id1 int, id2 int, primary key (id1)) comment 'vtocc_nocache' +create unique index id2_idx on upsert_test(id2) begin delete from vtocc_a delete from vtocc_c insert into vtocc_a(eid, id, name, foo) values(1, 1, 'abcd', 'efgh'), (1, 2, 'bcde', 'fghi') insert into vtocc_b(eid, id) values(1, 1), (1, 2) insert into vtocc_c(eid, name, foo) values(10, 'abcd', '20'), (11, 'bcde', '30') +delete from upsert_test commit create table vtocc_cached1(eid bigint, name varchar(128), foo varbinary(128), primary key(eid)) @@ -77,6 +80,7 @@ drop table if exists vtocc_c drop table if exists vtocc_d drop table if exists vtocc_e drop table if exists vtocc_f +drop table if exists upsert_test drop table if exists vtocc_cached1 drop table if exists vtocc_cached2 drop table if exists vtocc_renamed From 4aba6f3b7a67316f6171c4d10037beaeeb79bac5 Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Fri, 14 Aug 2015 23:02:44 -0700 Subject: [PATCH 03/80] tabletserver: upsert handles dup key error better Found a good way to know which index caused a dup key error. The error message itself contains that info. This is better than looking at RowsAffected because there are other conditions that cause that field to be 0. --- go/vt/tabletserver/query_executor.go | 12 +++++++++--- go/vt/tabletserver/query_executor_test.go | 8 ++++++-- test/queryservice_tests/cache_cases2.py | 21 +++++++++++++++++++++ test/queryservice_tests/nocache_cases.py | 11 +++++++++-- 4 files changed, 45 insertions(+), 7 deletions(-) diff --git a/go/vt/tabletserver/query_executor.go b/go/vt/tabletserver/query_executor.go index 1a0290c20d..93cecfe888 100644 --- a/go/vt/tabletserver/query_executor.go +++ b/go/vt/tabletserver/query_executor.go @@ -6,6 +6,7 @@ package tabletserver import ( "fmt" + "strings" "time" log "github.com/golang/glog" @@ -492,14 +493,19 @@ func (qre *QueryExecutor) execUpsertPK(conn poolConn, invalidator CacheInvalidat if terr.SqlError != mysql.ErrDupEntry { return nil, err } + // If the error didn't match pk, just return the error without updating. + if !strings.Contains(terr.Message, "'PRIMARY'") { + return nil, err + } result, err = qre.execDMLPKRows(conn, qre.plan.UpsertQuery, pkRows, invalidator) if err != nil { return nil, err } - if result.RowsAffected != 1 { - return nil, NewTabletError(ErrFail, "upsert failed to update a dup key row") + // Follow MySQL convention. RowsAffected must be 2 if a row was updated. + if result.RowsAffected == 1 { + result.RowsAffected = 2 } - return result, nil + return result, err } func (qre *QueryExecutor) execDMLPK(conn poolConn, invalidator CacheInvalidator) (*mproto.QueryResult, error) { diff --git a/go/vt/tabletserver/query_executor_test.go b/go/vt/tabletserver/query_executor_test.go index bcd0fa3db3..3b6b356a45 100644 --- a/go/vt/tabletserver/query_executor_test.go +++ b/go/vt/tabletserver/query_executor_test.go @@ -249,11 +249,15 @@ func TestQueryExecutorPlanUpsertPk(t *testing.T) { ) db.AddQuery("update test_table set val = 1 where pk in (1) /* _stream test_table (pk ) (1 ); */", &mproto.QueryResult{}) _, err = qre.Execute() - wantErr = "error: upsert failed to update a dup key row" + wantErr = "error: err (errno 1062)" if err == nil || err.Error() != wantErr { t.Fatalf("qre.Execute() = %v, want %v", err, wantErr) } + db.AddRejectedQuery( + "insert into test_table values (1) /* _stream test_table (pk ) (1 ); */", + sqldb.NewSqlError(mysql.ErrDupEntry, "ERROR 1062 (23000): Duplicate entry '2' for key 'PRIMARY'"), + ) db.AddQuery( "update test_table set val = 1 where pk in (1) /* _stream test_table (pk ) (1 ); */", &mproto.QueryResult{RowsAffected: 1}, @@ -263,7 +267,7 @@ func TestQueryExecutorPlanUpsertPk(t *testing.T) { t.Fatalf("qre.Execute() = %v, want nil", err) } want = &mproto.QueryResult{ - RowsAffected: 1, + RowsAffected: 2, } if !reflect.DeepEqual(got, want) { t.Fatalf("got: %v, want: %v", got, want) diff --git a/test/queryservice_tests/cache_cases2.py b/test/queryservice_tests/cache_cases2.py index 0b5130cbf0..f11cdb32a6 100644 --- a/test/queryservice_tests/cache_cases2.py +++ b/test/queryservice_tests/cache_cases2.py @@ -161,6 +161,27 @@ cases = [ cache_hits=1)]), # (1.foo, 1.bar, 2.foo, 2.bar) + MultiCase( + "upsert should invalidate rowcache", + [Case2(sql="select * from vtocc_cached2 where eid = 1 and bid = 'bar'", + result=[(1L, 'bar', 'abcd1', 'fghi')], + rowcount=1, + rewritten=[], + cache_hits=1), + 'begin', + Case2(sql="insert into vtocc_cached2 values(1, 'bar', 'abcd1', 'fghi') on duplicate key update foo='fghi'", + rowcount=0, + rewritten=[ + "insert into vtocc_cached2 values (1, 'bar', 'abcd1', 'fghi') /* _stream vtocc_cached2 (eid bid ) (1 'YmFy' )", + "update vtocc_cached2 set foo = 'fghi' where (eid = 1 and bid = 'bar') /* _stream vtocc_cached2 (eid bid ) (1 'YmFy' )"]), + 'commit', + Case2(sql="select * from vtocc_cached2 where eid = 1 and bid = 'bar'", + result=[(1L, 'bar', 'abcd1', 'fghi')], + rowcount=1, + rewritten=["select eid, bid, name, foo from vtocc_cached2 where (eid = 1 and bid = 'bar')"], + cache_misses=1)]), + # (1.foo, 1.bar, 2.foo, 2.bar) + MultiCase( "delete", ['begin', diff --git a/test/queryservice_tests/nocache_cases.py b/test/queryservice_tests/nocache_cases.py index 16d27c6949..ccc3e112ac 100644 --- a/test/queryservice_tests/nocache_cases.py +++ b/test/queryservice_tests/nocache_cases.py @@ -423,7 +423,14 @@ cases = [ rewritten=[ "insert into upsert_test(id1, id2) values (1, 2) /* _stream upsert_test (id1 ) (1 )", "update upsert_test set id2 = 2 where id1 in (1) /* _stream upsert_test (id1 ) (1 )"], - rowcount=1), + rowcount=2), + Case(sql='select * from upsert_test', + result=[(1L, 2L)]), + Case(sql="insert into upsert_test(id1, id2) values (1, 2) on duplicate key update id2 = 2", + rewritten=[ + "insert into upsert_test(id1, id2) values (1, 2) /* _stream upsert_test (id1 ) (1 )", + "update upsert_test set id2 = 2 where id1 in (1) /* _stream upsert_test (id1 ) (1 )"], + rowcount=0), Case(sql='select * from upsert_test', result=[(1L, 2L)]), 'commit', @@ -443,7 +450,7 @@ cases = [ rewritten=[ "insert into upsert_test(id1, id2) values (1, 2) /* _stream upsert_test (id1 ) (1 )", "update upsert_test set id1 = 2 where id1 in (1) /* _stream upsert_test (id1 ) (1 ) (2 )"], - rowcount=1), + rowcount=2), Case(sql='select * from upsert_test', result=[(2L, 1L)]), 'commit', From f788a6f1d743ca00abc05bba43a013e9d4e13892 Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Sat, 15 Aug 2015 15:14:10 -0700 Subject: [PATCH 04/80] tabletserver: upsert disallow values function This is a more correct fix for disallowing values function. I've verified that this function is only supported for the upsert construct: https://dev.mysql.com/doc/refman/5.5/en/miscellaneous-functions.html#function_values. So, it's safe to disallow this at the parser level, just as we do for last_insert_id. --- data/test/sqlparser_test/case_sensitivity.sql | 1 - data/test/sqlparser_test/parse_fail.sql | 2 +- data/test/sqlparser_test/parse_pass.sql | 2 +- go/vt/sqlparser/sql.go | 845 +++++++++--------- go/vt/sqlparser/sql.y | 8 +- go/vt/sqlparser/token.go | 1 - test/queryservice_tests/nocache_tests.py | 6 - 7 files changed, 422 insertions(+), 443 deletions(-) diff --git a/data/test/sqlparser_test/case_sensitivity.sql b/data/test/sqlparser_test/case_sensitivity.sql index d05b07a8bf..7ced59224d 100644 --- a/data/test/sqlparser_test/case_sensitivity.sql +++ b/data/test/sqlparser_test/case_sensitivity.sql @@ -17,7 +17,6 @@ select A() from b#select a() from b select A(B, C) from b#select a(b, c) from b select A(distinct B, C) from b#select a(distinct b, c) from b select IF(B, C) from b#select if(b, c) from b -select VALUES(B, C) from b#select values(b, c) from b select * from b use index (A)#select * from b use index (a) insert into A(A, B) values (1, 2)#insert into A(a, b) values (1, 2) CREATE TABLE A#create table A diff --git a/data/test/sqlparser_test/parse_fail.sql b/data/test/sqlparser_test/parse_fail.sql index ae92ff7427..bf9c1341ef 100644 --- a/data/test/sqlparser_test/parse_fail.sql +++ b/data/test/sqlparser_test/parse_fail.sql @@ -11,7 +11,7 @@ select * from t where :1 = 2#syntax error at position 24 near : select * from t where :. = 2#syntax error at position 24 near : select * from t where ::1 = 2#syntax error at position 25 near :: select * from t where ::. = 2#syntax error at position 25 near :: -update a set c = value(1)#syntax error at position 23 near value +update a set c = values(1)#syntax error at position 24 near values update a set c = last_insert_id(1)#syntax error at position 32 near last_insert_id select(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(#max nesting level reached at position 406 select(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(#syntax error at position 405 diff --git a/data/test/sqlparser_test/parse_pass.sql b/data/test/sqlparser_test/parse_pass.sql index 8458c098ad..b62ea4bdb8 100644 --- a/data/test/sqlparser_test/parse_pass.sql +++ b/data/test/sqlparser_test/parse_pass.sql @@ -155,7 +155,7 @@ insert /* value expression list */ into a values (a + 1, 2 * 3) insert /* column list */ into a(a, b) values (1, 2) insert /* qualified column list */ into a(a, a.b) values (1, 2) insert /* select */ into a select b, c from d -insert /* on duplicate */ into a values (1, 2) on duplicate key update b = values(a), c = d +insert /* on duplicate */ into a values (1, 2) on duplicate key update b = func(a), c = d update /* simple */ a set b = 3 update /* a.b */ a.b set b = 3 update /* b.c */ a set b.c = 3 diff --git a/go/vt/sqlparser/sql.go b/go/vt/sqlparser/sql.go index 3d928afa77..36025c9d1b 100644 --- a/go/vt/sqlparser/sql.go +++ b/go/vt/sqlparser/sql.go @@ -94,68 +94,67 @@ const BETWEEN = 57366 const NULL = 57367 const ASC = 57368 const DESC = 57369 -const VALUES = 57370 -const INTO = 57371 -const DUPLICATE = 57372 -const KEY = 57373 -const DEFAULT = 57374 -const SET = 57375 -const LOCK = 57376 -const KEYRANGE = 57377 -const VALUE = 57378 -const LAST_INSERT_ID = 57379 -const ID = 57380 -const STRING = 57381 -const NUMBER = 57382 -const VALUE_ARG = 57383 -const LIST_ARG = 57384 -const COMMENT = 57385 -const LE = 57386 -const GE = 57387 -const NE = 57388 -const NULL_SAFE_EQUAL = 57389 -const UNION = 57390 -const MINUS = 57391 -const EXCEPT = 57392 -const INTERSECT = 57393 -const JOIN = 57394 -const STRAIGHT_JOIN = 57395 -const LEFT = 57396 -const RIGHT = 57397 -const INNER = 57398 -const OUTER = 57399 -const CROSS = 57400 -const NATURAL = 57401 -const USE = 57402 -const FORCE = 57403 -const ON = 57404 -const OR = 57405 -const AND = 57406 -const NOT = 57407 -const SHIFT_LEFT = 57408 -const SHIFT_RIGHT = 57409 -const UNARY = 57410 -const CASE = 57411 -const WHEN = 57412 -const THEN = 57413 -const ELSE = 57414 -const END = 57415 -const CREATE = 57416 -const ALTER = 57417 -const DROP = 57418 -const RENAME = 57419 -const ANALYZE = 57420 -const TABLE = 57421 -const INDEX = 57422 -const VIEW = 57423 -const TO = 57424 -const IGNORE = 57425 -const IF = 57426 -const UNIQUE = 57427 -const USING = 57428 -const SHOW = 57429 -const DESCRIBE = 57430 -const EXPLAIN = 57431 +const INTO = 57370 +const DUPLICATE = 57371 +const KEY = 57372 +const DEFAULT = 57373 +const SET = 57374 +const LOCK = 57375 +const KEYRANGE = 57376 +const VALUES = 57377 +const LAST_INSERT_ID = 57378 +const ID = 57379 +const STRING = 57380 +const NUMBER = 57381 +const VALUE_ARG = 57382 +const LIST_ARG = 57383 +const COMMENT = 57384 +const LE = 57385 +const GE = 57386 +const NE = 57387 +const NULL_SAFE_EQUAL = 57388 +const UNION = 57389 +const MINUS = 57390 +const EXCEPT = 57391 +const INTERSECT = 57392 +const JOIN = 57393 +const STRAIGHT_JOIN = 57394 +const LEFT = 57395 +const RIGHT = 57396 +const INNER = 57397 +const OUTER = 57398 +const CROSS = 57399 +const NATURAL = 57400 +const USE = 57401 +const FORCE = 57402 +const ON = 57403 +const OR = 57404 +const AND = 57405 +const NOT = 57406 +const SHIFT_LEFT = 57407 +const SHIFT_RIGHT = 57408 +const UNARY = 57409 +const CASE = 57410 +const WHEN = 57411 +const THEN = 57412 +const ELSE = 57413 +const END = 57414 +const CREATE = 57415 +const ALTER = 57416 +const DROP = 57417 +const RENAME = 57418 +const ANALYZE = 57419 +const TABLE = 57420 +const INDEX = 57421 +const VIEW = 57422 +const TO = 57423 +const IGNORE = 57424 +const IF = 57425 +const UNIQUE = 57426 +const USING = 57427 +const SHOW = 57428 +const DESCRIBE = 57429 +const EXPLAIN = 57430 var yyToknames = [...]string{ "$end", @@ -185,7 +184,6 @@ var yyToknames = [...]string{ "NULL", "ASC", "DESC", - "VALUES", "INTO", "DUPLICATE", "KEY", @@ -193,7 +191,7 @@ var yyToknames = [...]string{ "SET", "LOCK", "KEYRANGE", - "VALUE", + "VALUES", "LAST_INSERT_ID", "ID", "STRING", @@ -276,147 +274,146 @@ var yyExca = [...]int{ 1, -1, -2, 0, -1, 68, - 83, 203, - -2, 202, + 82, 202, + -2, 201, } -const yyNprod = 207 +const yyNprod = 206 const yyPrivate = 57344 var yyTokenNames []string var yyStates []string -const yyLast = 732 +const yyLast = 722 var yyAct = [...]int{ - 98, 166, 163, 340, 236, 373, 257, 93, 303, 63, - 94, 215, 295, 204, 92, 165, 3, 382, 184, 132, - 64, 248, 138, 137, 237, 82, 192, 83, 78, 239, - 301, 70, 108, 38, 43, 40, 44, 14, 66, 41, - 264, 72, 126, 53, 75, 65, 351, 268, 269, 270, - 271, 272, 350, 273, 274, 349, 74, 107, 88, 237, - 110, 71, 49, 320, 322, 45, 237, 87, 237, 331, - 68, 104, 105, 106, 28, 29, 30, 31, 237, 237, - 103, 130, 46, 47, 48, 249, 134, 293, 249, 279, - 136, 138, 137, 119, 167, 237, 321, 115, 168, 170, - 171, 147, 153, 154, 147, 95, 96, 198, 121, 333, - 73, 138, 137, 97, 178, 66, 111, 224, 66, 137, - 188, 187, 65, 346, 182, 65, 196, 237, 296, 260, - 117, 109, 199, 129, 174, 88, 210, 188, 186, 314, - 50, 296, 214, 348, 315, 222, 223, 211, 226, 227, - 228, 229, 230, 231, 232, 233, 234, 235, 162, 164, - 209, 347, 225, 312, 208, 169, 51, 52, 313, 318, - 238, 240, 241, 217, 317, 88, 88, 242, 59, 316, - 117, 66, 66, 239, 195, 197, 194, 131, 65, 255, - 253, 246, 185, 261, 185, 252, 358, 335, 243, 245, - 290, 256, 118, 103, 61, 212, 213, 150, 151, 152, - 153, 154, 147, 123, 103, 112, 125, 73, 280, 241, - 278, 91, 265, 282, 283, 68, 268, 269, 270, 271, - 272, 281, 273, 274, 132, 77, 180, 113, 266, 208, - 117, 116, 286, 28, 29, 30, 31, 88, 287, 14, - 289, 103, 217, 259, 218, 61, 91, 91, 277, 300, - 103, 189, 298, 302, 172, 173, 299, 175, 176, 292, - 288, 202, 122, 360, 361, 107, 135, 61, 79, 181, - 310, 311, 61, 324, 80, 326, 114, 328, 334, 104, - 105, 106, 103, 329, 379, 73, 332, 206, 91, 208, - 208, 355, 66, 91, 91, 330, 216, 338, 341, 336, - 14, 337, 380, 58, 285, 386, 190, 294, 146, 145, - 148, 149, 150, 151, 152, 153, 154, 147, 56, 219, - 352, 220, 221, 251, 262, 353, 354, 128, 91, 91, - 54, 304, 345, 305, 258, 344, 356, 364, 241, 309, - 362, 91, 148, 149, 150, 151, 152, 153, 154, 147, - 370, 341, 371, 185, 62, 385, 369, 374, 374, 374, - 66, 372, 206, 375, 376, 67, 14, 65, 381, 377, - 383, 384, 33, 387, 275, 216, 133, 388, 191, 389, - 39, 14, 15, 16, 17, 146, 145, 148, 149, 150, - 151, 152, 153, 154, 147, 263, 244, 193, 101, 42, - 91, 60, 363, 107, 365, 91, 110, 69, 254, 18, - 179, 76, 378, 102, 32, 81, 68, 104, 105, 106, - 359, 86, 206, 206, 60, 339, 103, 343, 60, 308, - 34, 35, 36, 37, 291, 120, 177, 342, 247, 100, - 124, 99, 297, 127, 250, 139, 89, 319, 90, 205, - 267, 95, 96, 84, 203, 85, 55, 27, 57, 97, - 13, 12, 111, 11, 10, 19, 20, 22, 21, 23, - 9, 107, 8, 7, 110, 6, 5, 109, 24, 25, - 26, 183, 4, 237, 68, 104, 105, 106, 2, 1, - 0, 0, 200, 0, 103, 201, 101, 207, 86, 0, - 91, 107, 91, 0, 110, 366, 367, 368, 0, 0, - 0, 102, 0, 0, 68, 104, 105, 106, 0, 95, - 96, 0, 0, 0, 103, 0, 0, 97, 0, 0, - 111, 0, 0, 14, 0, 0, 0, 0, 86, 86, - 0, 0, 0, 0, 0, 109, 90, 0, 101, 95, - 96, 84, 0, 107, 0, 0, 110, 97, 0, 0, - 111, 0, 0, 102, 0, 0, 68, 104, 105, 106, - 0, 276, 207, 0, 0, 109, 103, 0, 0, 0, - 0, 0, 0, 0, 0, 101, 0, 0, 0, 0, - 107, 0, 0, 110, 0, 0, 0, 0, 90, 357, - 102, 95, 96, 68, 104, 105, 106, 0, 0, 97, - 86, 0, 111, 103, 146, 145, 148, 149, 150, 151, - 152, 153, 154, 147, 0, 0, 306, 109, 0, 307, - 0, 0, 207, 207, 0, 90, 0, 0, 95, 96, - 0, 0, 0, 323, 0, 325, 97, 0, 0, 111, - 140, 144, 142, 143, 145, 148, 149, 150, 151, 152, - 153, 154, 147, 0, 109, 0, 0, 0, 0, 0, - 0, 0, 0, 158, 159, 160, 161, 0, 155, 156, - 157, 327, 0, 146, 145, 148, 149, 150, 151, 152, - 153, 154, 147, 0, 0, 0, 0, 0, 0, 141, - 146, 145, 148, 149, 150, 151, 152, 153, 154, 147, - 284, 0, 146, 145, 148, 149, 150, 151, 152, 153, - 154, 147, + 98, 165, 162, 339, 235, 372, 256, 93, 302, 63, + 94, 82, 247, 203, 92, 214, 294, 183, 164, 3, + 64, 236, 267, 268, 269, 270, 271, 83, 272, 273, + 381, 28, 29, 30, 31, 107, 137, 136, 66, 131, + 238, 72, 50, 300, 75, 65, 53, 68, 104, 105, + 106, 191, 78, 38, 70, 40, 350, 103, 88, 41, + 263, 125, 319, 321, 74, 43, 87, 44, 51, 52, + 236, 349, 348, 236, 49, 108, 46, 47, 48, 236, + 71, 129, 95, 96, 236, 45, 133, 330, 236, 236, + 97, 248, 236, 110, 166, 320, 278, 135, 167, 169, + 170, 248, 118, 292, 137, 136, 114, 146, 109, 152, + 153, 146, 120, 177, 66, 122, 223, 66, 124, 187, + 186, 65, 332, 181, 65, 144, 147, 148, 149, 150, + 151, 152, 153, 146, 88, 209, 187, 185, 137, 136, + 136, 213, 345, 295, 221, 222, 259, 225, 226, 227, + 228, 229, 230, 231, 232, 233, 234, 161, 163, 208, + 224, 197, 188, 73, 128, 168, 347, 116, 116, 237, + 239, 240, 201, 346, 88, 88, 241, 173, 295, 195, + 66, 66, 238, 242, 244, 198, 317, 65, 254, 252, + 245, 59, 260, 313, 316, 315, 311, 251, 314, 255, + 210, 312, 130, 211, 212, 357, 207, 184, 184, 356, + 28, 29, 30, 31, 77, 216, 334, 279, 240, 277, + 264, 91, 281, 282, 145, 144, 147, 148, 149, 150, + 151, 152, 153, 146, 280, 261, 289, 194, 196, 193, + 117, 285, 103, 14, 111, 73, 88, 286, 131, 288, + 112, 258, 265, 116, 115, 287, 91, 91, 299, 291, + 61, 297, 301, 80, 171, 172, 68, 174, 175, 298, + 103, 217, 359, 360, 179, 61, 61, 103, 180, 309, + 310, 207, 323, 378, 325, 103, 327, 121, 79, 103, + 113, 276, 328, 354, 216, 331, 205, 91, 107, 333, + 379, 66, 91, 91, 329, 215, 337, 340, 335, 61, + 336, 104, 105, 106, 134, 293, 145, 144, 147, 148, + 149, 150, 151, 152, 153, 146, 58, 284, 218, 351, + 219, 220, 73, 14, 352, 353, 189, 91, 91, 385, + 127, 207, 207, 56, 54, 355, 363, 240, 341, 361, + 91, 147, 148, 149, 150, 151, 152, 153, 146, 369, + 340, 370, 303, 250, 344, 67, 373, 373, 373, 66, + 371, 205, 374, 375, 304, 257, 65, 380, 376, 382, + 383, 343, 386, 308, 215, 184, 387, 62, 388, 149, + 150, 151, 152, 153, 146, 384, 368, 243, 14, 101, + 33, 60, 274, 132, 107, 190, 39, 262, 192, 91, + 362, 76, 364, 102, 91, 81, 68, 104, 105, 106, + 42, 86, 69, 253, 60, 178, 103, 377, 60, 358, + 338, 205, 205, 342, 307, 119, 14, 15, 16, 17, + 123, 290, 176, 126, 246, 100, 99, 296, 90, 249, + 138, 95, 96, 84, 267, 268, 269, 270, 271, 97, + 272, 273, 110, 18, 326, 89, 145, 144, 147, 148, + 149, 150, 151, 152, 153, 146, 318, 109, 204, 266, + 182, 202, 85, 236, 55, 27, 57, 13, 12, 11, + 10, 199, 9, 8, 200, 7, 206, 86, 145, 144, + 147, 148, 149, 150, 151, 152, 153, 146, 6, 91, + 5, 91, 4, 2, 365, 366, 367, 1, 0, 19, + 20, 22, 21, 23, 0, 101, 14, 0, 0, 0, + 107, 32, 24, 25, 26, 0, 0, 86, 86, 102, + 0, 101, 68, 104, 105, 106, 107, 34, 35, 36, + 37, 0, 103, 0, 0, 102, 0, 0, 68, 104, + 105, 106, 0, 0, 0, 0, 0, 0, 103, 0, + 275, 206, 0, 0, 90, 0, 0, 95, 96, 84, + 0, 0, 0, 0, 0, 97, 0, 0, 110, 0, + 90, 0, 0, 95, 96, 0, 0, 101, 0, 0, + 0, 97, 107, 109, 110, 0, 14, 0, 0, 86, + 0, 102, 0, 0, 68, 104, 105, 106, 0, 109, + 0, 0, 0, 0, 103, 305, 107, 0, 306, 0, + 0, 206, 206, 0, 0, 0, 0, 0, 68, 104, + 105, 106, 322, 0, 324, 0, 90, 0, 103, 95, + 96, 0, 0, 0, 0, 0, 0, 97, 0, 0, + 110, 0, 0, 139, 143, 141, 142, 0, 0, 0, + 0, 0, 0, 95, 96, 109, 0, 0, 0, 0, + 0, 97, 0, 0, 110, 157, 158, 159, 160, 0, + 154, 155, 156, 0, 0, 0, 0, 0, 283, 109, + 145, 144, 147, 148, 149, 150, 151, 152, 153, 146, + 0, 140, 145, 144, 147, 148, 149, 150, 151, 152, + 153, 146, } var yyPact = [...]int{ - 386, -1000, -1000, 191, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -61, - -62, -29, -12, -32, -1000, -1000, -1000, 371, 323, -1000, - -1000, -1000, 310, -1000, 284, 217, 355, 187, -68, -34, - 179, -1000, -38, 179, -1000, 217, -71, 240, -71, 217, - -1000, -1000, -1000, -1000, -1000, 486, -1000, 172, 217, 253, - 14, -1000, 217, 124, -1000, 153, -1000, 10, -1000, 217, - 38, 234, -1000, -1000, 217, -1000, -55, 217, 317, 66, - 179, -1000, 178, -1000, -1000, 257, 7, 43, 639, -1000, - 575, 538, -1000, -1000, -1000, 456, 456, 456, 155, 155, - -1000, 155, 155, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, 456, -1000, 203, 187, 217, 353, 187, 456, 179, - -1000, 296, -75, -1000, 94, -1000, 217, -1000, -1000, 217, - -1000, 166, 486, -1000, -1000, 179, 72, 575, 575, 456, - 212, 308, 456, 456, 92, 456, 456, 456, 456, 456, - 456, 456, 456, 456, 456, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, 639, -46, 22, -27, 639, -1000, 32, - -1000, -1000, 388, 486, -1000, 371, 250, 3, 324, 305, - 187, 187, 184, -1000, 331, 575, -1000, 324, -1000, -1000, - -1000, 62, 179, -1000, -57, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, 182, 169, 239, 244, 6, -1000, -1000, - -1000, -1000, -1000, 50, 324, -1000, 32, -1000, -1000, 212, - 456, 456, 324, 651, -1000, 289, 279, 592, -1000, 132, - 132, 24, 24, 24, 21, 21, -1000, -1000, -1000, 456, - -1000, 324, -1000, -37, 486, -37, 144, 0, -1000, 575, - 61, 155, 191, 74, -26, -1000, 331, 326, 329, 43, - 217, -1000, -1000, 217, -1000, 338, 166, 166, -1000, -1000, - 106, 82, 122, 117, 112, -2, -1000, 217, -10, 217, - -27, -1000, 324, 622, 456, -1000, 324, -1000, -37, -1000, - 250, -19, -1000, 456, 23, -1000, 258, 141, -1000, -1000, - -1000, 187, 326, -1000, 456, 456, -1000, -1000, 333, 328, - 169, 56, -1000, 104, -1000, 86, -1000, -1000, -1000, -1000, - -40, -43, -49, -1000, -1000, -1000, -1000, 456, 324, -1000, - -81, -1000, 324, 456, 270, 155, -1000, -1000, 553, 140, - -1000, 247, -1000, 331, 575, 456, 575, -1000, -1000, 155, - 155, 155, 324, -1000, 324, 359, -1000, 456, 456, -1000, - -1000, -1000, 326, 43, 127, 43, 179, 179, 179, 187, - 324, -1000, 278, -39, -1000, -39, -39, 124, -1000, 358, - 294, -1000, 179, -1000, -1000, -1000, 179, -1000, 179, -1000, + 431, -1000, -1000, 159, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -40, + -30, -8, -17, -19, -1000, -1000, -1000, 393, 327, -1000, + -1000, -1000, 325, -1000, 298, 239, 378, 229, -44, -14, + 208, -1000, -29, 208, -1000, 239, -46, 251, -46, 239, + -1000, -1000, -1000, -1000, -1000, 505, -1000, 202, 239, 258, + 24, -1000, 239, 113, -1000, 192, -1000, 20, -1000, 239, + 43, 250, -1000, -1000, 239, -1000, -35, 239, 320, 98, + 208, -1000, 193, -1000, -1000, 295, 15, 71, 642, -1000, + 577, 521, -1000, -1000, -1000, 10, 10, 10, 195, 195, + -1000, 195, 195, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + 10, -1000, 242, 229, 239, 375, 229, 10, 208, -1000, + 316, -49, -1000, 148, -1000, 239, -1000, -1000, 239, -1000, + 223, 505, -1000, -1000, 208, 126, 577, 577, 10, 230, + 307, 10, 10, 91, 10, 10, 10, 10, 10, 10, + 10, 10, 10, 10, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, 642, -31, -20, -15, 642, -1000, 601, -1000, + -1000, 379, 505, -1000, 393, 273, 7, 428, 328, 229, + 229, 198, -1000, 362, 577, -1000, 428, -1000, -1000, -1000, + 80, 208, -1000, -36, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, 197, 398, 272, 238, 14, -1000, -1000, -1000, + -1000, -1000, 72, 428, -1000, 601, -1000, -1000, 230, 10, + 10, 428, 630, -1000, 302, 279, 54, -1000, 315, 315, + 32, 32, 32, 28, 28, -1000, -1000, -1000, 10, -1000, + 428, -1000, -16, 505, -16, 181, 17, -1000, 577, 77, + 195, 159, 112, -12, -1000, 362, 347, 360, 71, 239, + -1000, -1000, 239, -1000, 372, 223, 223, -1000, -1000, 140, + 137, 139, 138, 130, -2, -1000, 239, -34, 239, -15, + -1000, 428, 396, 10, -1000, 428, -1000, -16, -1000, 273, + 0, -1000, 10, 37, -1000, 270, 161, -1000, -1000, -1000, + 229, 347, -1000, 10, 10, -1000, -1000, 369, 350, 398, + 76, -1000, 117, -1000, 110, -1000, -1000, -1000, -1000, -22, + -23, -38, -1000, -1000, -1000, -1000, 10, 428, -1000, -83, + -1000, 428, 10, 263, 195, -1000, -1000, 154, 150, -1000, + 246, -1000, 362, 577, 10, 577, -1000, -1000, 195, 195, + 195, 428, -1000, 428, 389, -1000, 10, 10, -1000, -1000, + -1000, 347, 71, 127, 71, 208, 208, 208, 229, 428, + -1000, 267, -25, -1000, -25, -25, 113, -1000, 388, 318, + -1000, 208, -1000, -1000, -1000, 208, -1000, 208, -1000, } var yyPgo = [...]int{ - 0, 499, 498, 15, 492, 486, 485, 483, 482, 480, - 474, 473, 471, 470, 424, 468, 467, 466, 25, 27, - 465, 464, 13, 460, 459, 178, 457, 5, 18, 67, - 456, 455, 454, 14, 2, 11, 1, 452, 10, 451, - 32, 7, 449, 448, 21, 446, 444, 439, 437, 6, - 435, 3, 430, 8, 422, 420, 418, 12, 9, 20, - 235, 417, 409, 407, 405, 390, 388, 0, 386, 375, - 384, 140, 382, 165, 4, + 0, 517, 513, 18, 512, 510, 508, 495, 493, 492, + 490, 489, 488, 487, 531, 486, 485, 484, 11, 27, + 482, 481, 13, 479, 478, 191, 476, 5, 17, 66, + 465, 450, 449, 14, 2, 15, 1, 447, 10, 446, + 75, 7, 445, 444, 12, 442, 441, 434, 433, 6, + 430, 3, 429, 8, 427, 425, 423, 16, 9, 20, + 214, 422, 420, 408, 407, 406, 405, 0, 403, 365, + 402, 42, 400, 165, 4, } var yyR1 = [...]int{ @@ -433,14 +430,14 @@ var yyR1 = [...]int{ 30, 30, 31, 31, 31, 31, 31, 31, 31, 35, 35, 35, 40, 36, 36, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 39, 39, 42, 45, - 45, 43, 43, 44, 46, 46, 41, 41, 33, 33, - 33, 33, 47, 47, 48, 48, 49, 49, 50, 50, - 51, 52, 52, 52, 53, 53, 53, 54, 54, 54, - 55, 55, 56, 56, 57, 57, 32, 32, 37, 37, - 38, 38, 58, 58, 59, 60, 60, 61, 61, 62, - 62, 63, 63, 63, 63, 63, 64, 64, 65, 65, - 66, 66, 67, 69, 73, 74, 71, + 34, 34, 34, 34, 34, 34, 39, 42, 45, 45, + 43, 43, 44, 46, 46, 41, 41, 33, 33, 33, + 33, 47, 47, 48, 48, 49, 49, 50, 50, 51, + 52, 52, 52, 53, 53, 53, 54, 54, 54, 55, + 55, 56, 56, 57, 57, 32, 32, 37, 37, 38, + 38, 58, 58, 59, 60, 60, 61, 61, 62, 62, + 63, 63, 63, 63, 63, 64, 64, 65, 65, 66, + 66, 67, 69, 73, 74, 71, } var yyR2 = [...]int{ @@ -457,114 +454,114 @@ var yyR2 = [...]int{ 2, 6, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 3, 1, 3, 1, 1, 1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, - 2, 3, 4, 5, 4, 1, 1, 1, 5, 0, - 1, 1, 2, 4, 0, 2, 1, 3, 1, 1, - 1, 1, 0, 3, 0, 2, 0, 3, 1, 3, - 2, 0, 1, 1, 0, 2, 4, 0, 2, 4, - 0, 3, 1, 3, 0, 5, 2, 1, 1, 3, - 3, 1, 1, 3, 3, 0, 2, 0, 3, 0, - 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, - 0, 2, 1, 1, 1, 1, 0, + 2, 3, 4, 5, 4, 1, 1, 5, 0, 1, + 1, 2, 4, 0, 2, 1, 3, 1, 1, 1, + 1, 0, 3, 0, 2, 0, 3, 1, 3, 2, + 0, 1, 1, 0, 2, 4, 0, 2, 4, 0, + 3, 1, 3, 0, 5, 2, 1, 1, 3, 3, + 1, 1, 3, 3, 0, 2, 0, 3, 0, 1, + 1, 1, 1, 1, 1, 0, 1, 0, 1, 0, + 2, 1, 1, 1, 1, 0, } var yyChk = [...]int{ -1000, -1, -2, -3, -4, -5, -6, -7, -8, -9, - -10, -11, -12, -13, 5, 6, 7, 8, 33, 89, - 90, 92, 91, 93, 102, 103, 104, -16, 52, 53, - 54, 55, -14, -72, -14, -14, -14, -14, 94, -65, - 96, 100, -62, 96, 98, 94, 94, 95, 96, 94, - -71, -71, -71, -3, 17, -17, 18, -15, 29, -25, - -69, 38, 9, -58, -59, -41, -67, -69, 38, -61, - 99, 95, -67, 38, 94, -67, -69, -60, 99, 38, - -60, -69, -18, -19, 75, -20, -69, -29, -34, -30, - 70, -73, -33, -41, -38, 73, 74, 81, -67, -39, - -42, 20, 35, 48, 39, 40, 41, 25, -40, 99, - 28, 84, 43, -25, 33, 83, -25, 56, 49, 83, - -69, 70, 38, -71, -69, -71, 97, -69, 20, 67, - -67, 9, 56, -68, -67, 19, 83, 69, 68, -31, - 21, 70, 23, 24, 22, 72, 71, 80, 73, 74, - 75, 76, 77, 78, 79, 49, 50, 51, 44, 45, - 46, 47, -29, -34, -29, -3, -36, -34, -34, -73, - -34, -34, -73, -73, -40, -73, -73, -45, -34, -55, - 33, -73, -58, -69, -28, 10, -59, -34, -67, -71, - 20, -66, 101, -63, 92, 90, 32, 91, 13, 38, - -69, -69, -71, -21, -22, -24, -73, -69, -40, -19, - -67, 75, -29, -29, -34, -35, -73, -40, 42, 21, - 23, 24, -34, -34, 25, 70, -34, -34, -34, -34, - -34, -34, -34, -34, -34, -34, -74, 105, -74, 56, - -74, -34, -74, -18, 18, -18, -33, -43, -44, 85, - -32, 28, -3, -58, -56, -41, -28, -49, 13, -29, - 67, -67, -71, -64, 97, -28, 56, -23, 57, 58, - 59, 60, 61, 63, 64, -70, -69, 19, -22, 83, - -36, -35, -34, -34, 69, 25, -34, -74, -18, -74, - 56, -46, -44, 87, -29, -57, 67, -37, -38, -57, - -74, 56, -49, -53, 15, 14, -69, -69, -47, 11, - -22, -22, 57, 62, 57, 62, 57, 57, 57, -26, - 65, 98, 66, -69, -74, -69, -74, 69, -34, -74, - -33, 88, -34, 86, 30, 56, -41, -53, -34, -50, - -51, -34, -71, -48, 12, 14, 67, 57, 57, 95, - 95, 95, -34, -74, -34, 31, -38, 56, 56, -52, - 26, 27, -49, -29, -36, -29, -73, -73, -73, 7, - -34, -51, -53, -27, -67, -27, -27, -58, -54, 16, - 34, -74, 56, -74, -74, 7, 21, -67, -67, -67, + -10, -11, -12, -13, 5, 6, 7, 8, 32, 88, + 89, 91, 90, 92, 101, 102, 103, -16, 51, 52, + 53, 54, -14, -72, -14, -14, -14, -14, 93, -65, + 95, 99, -62, 95, 97, 93, 93, 94, 95, 93, + -71, -71, -71, -3, 17, -17, 18, -15, 28, -25, + -69, 37, 9, -58, -59, -41, -67, -69, 37, -61, + 98, 94, -67, 37, 93, -67, -69, -60, 98, 37, + -60, -69, -18, -19, 74, -20, -69, -29, -34, -30, + 69, -73, -33, -41, -38, 72, 73, 80, -67, -39, + -42, 20, 34, 47, 38, 39, 40, 25, -40, 98, + 83, 42, -25, 32, 82, -25, 55, 48, 82, -69, + 69, 37, -71, -69, -71, 96, -69, 20, 66, -67, + 9, 55, -68, -67, 19, 82, 68, 67, -31, 21, + 69, 23, 24, 22, 71, 70, 79, 72, 73, 74, + 75, 76, 77, 78, 48, 49, 50, 43, 44, 45, + 46, -29, -34, -29, -3, -36, -34, -34, -73, -34, + -34, -73, -73, -40, -73, -73, -45, -34, -55, 32, + -73, -58, -69, -28, 10, -59, -34, -67, -71, 20, + -66, 100, -63, 91, 89, 31, 90, 13, 37, -69, + -69, -71, -21, -22, -24, -73, -69, -40, -19, -67, + 74, -29, -29, -34, -35, -73, -40, 41, 21, 23, + 24, -34, -34, 25, 69, -34, -34, -34, -34, -34, + -34, -34, -34, -34, -34, -74, 104, -74, 55, -74, + -34, -74, -18, 18, -18, -33, -43, -44, 84, -32, + 35, -3, -58, -56, -41, -28, -49, 13, -29, 66, + -67, -71, -64, 96, -28, 55, -23, 56, 57, 58, + 59, 60, 62, 63, -70, -69, 19, -22, 82, -36, + -35, -34, -34, 68, 25, -34, -74, -18, -74, 55, + -46, -44, 86, -29, -57, 66, -37, -38, -57, -74, + 55, -49, -53, 15, 14, -69, -69, -47, 11, -22, + -22, 56, 61, 56, 61, 56, 56, 56, -26, 64, + 97, 65, -69, -74, -69, -74, 68, -34, -74, -33, + 87, -34, 85, 29, 55, -41, -53, -34, -50, -51, + -34, -71, -48, 12, 14, 66, 56, 56, 94, 94, + 94, -34, -74, -34, 30, -38, 55, 55, -52, 26, + 27, -49, -29, -36, -29, -73, -73, -73, 7, -34, + -51, -53, -27, -67, -27, -27, -58, -54, 16, 33, + -74, 55, -74, -74, 7, 21, -67, -67, -67, } var yyDef = [...]int{ 0, -2, 1, 2, 3, 4, 5, 6, 7, 8, - 9, 10, 11, 12, 34, 34, 34, 34, 34, 198, - 189, 0, 0, 0, 206, 206, 206, 0, 38, 40, - 41, 42, 43, 36, 0, 0, 0, 0, 187, 0, - 0, 199, 0, 0, 190, 0, 185, 0, 185, 0, + 9, 10, 11, 12, 34, 34, 34, 34, 34, 197, + 188, 0, 0, 0, 205, 205, 205, 0, 38, 40, + 41, 42, 43, 36, 0, 0, 0, 0, 186, 0, + 0, 198, 0, 0, 189, 0, 184, 0, 184, 0, 31, 32, 33, 14, 39, 0, 44, 35, 0, 0, - 76, 203, 0, 19, 182, 0, 146, 0, -2, 0, - 0, 0, 206, 202, 0, 206, 0, 0, 0, 0, + 76, 202, 0, 19, 181, 0, 145, 0, -2, 0, + 0, 0, 205, 201, 0, 205, 0, 0, 0, 0, 0, 30, 0, 45, 47, 52, 0, 50, 51, 86, - 0, 0, 115, 116, 117, 0, 0, 0, 146, 0, - 135, 0, 0, 204, 148, 149, 150, 151, 181, 136, - 137, 139, 37, 170, 0, 0, 84, 0, 0, 0, - 206, 0, 200, 22, 0, 25, 0, 27, 186, 0, - 206, 0, 0, 48, 53, 0, 0, 0, 0, 0, + 0, 0, 115, 116, 117, 0, 0, 0, 145, 0, + 135, 0, 0, 203, 147, 148, 149, 150, 180, 136, + 138, 37, 169, 0, 0, 84, 0, 0, 0, 205, + 0, 199, 22, 0, 25, 0, 27, 185, 0, 205, + 0, 0, 48, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 102, 103, 104, 105, 106, - 107, 108, 89, 0, 0, 0, 0, 113, 128, 0, - 129, 130, 0, 0, 100, 0, 0, 0, 140, 0, - 0, 0, 84, 77, 156, 0, 183, 184, 147, 20, - 188, 0, 0, 206, 196, 191, 192, 193, 194, 195, - 26, 28, 29, 84, 55, 61, 0, 73, 75, 46, - 54, 49, 87, 88, 91, 92, 0, 110, 111, 0, - 0, 0, 94, 0, 98, 0, 118, 119, 120, 121, - 122, 123, 124, 125, 126, 127, 90, 205, 112, 0, - 180, 113, 131, 0, 0, 0, 0, 144, 141, 0, - 174, 0, 177, 174, 0, 172, 156, 164, 0, 85, - 0, 201, 23, 0, 197, 152, 0, 0, 64, 65, - 0, 0, 0, 0, 0, 78, 62, 0, 0, 0, - 0, 93, 95, 0, 0, 99, 114, 132, 0, 134, - 0, 0, 142, 0, 0, 15, 0, 176, 178, 16, - 171, 0, 164, 18, 0, 0, 206, 24, 154, 0, - 56, 59, 66, 0, 68, 0, 70, 71, 72, 57, - 0, 0, 0, 63, 58, 74, 109, 0, 96, 133, - 0, 138, 145, 0, 0, 0, 173, 17, 165, 157, - 158, 161, 21, 156, 0, 0, 0, 67, 69, 0, - 0, 0, 97, 101, 143, 0, 179, 0, 0, 160, - 162, 163, 164, 155, 153, 60, 0, 0, 0, 0, - 166, 159, 167, 0, 82, 0, 0, 175, 13, 0, - 0, 79, 0, 80, 81, 168, 0, 83, 0, 169, + 0, 0, 0, 0, 102, 103, 104, 105, 106, 107, + 108, 89, 0, 0, 0, 0, 113, 128, 0, 129, + 130, 0, 0, 100, 0, 0, 0, 139, 0, 0, + 0, 84, 77, 155, 0, 182, 183, 146, 20, 187, + 0, 0, 205, 195, 190, 191, 192, 193, 194, 26, + 28, 29, 84, 55, 61, 0, 73, 75, 46, 54, + 49, 87, 88, 91, 92, 0, 110, 111, 0, 0, + 0, 94, 0, 98, 0, 118, 119, 120, 121, 122, + 123, 124, 125, 126, 127, 90, 204, 112, 0, 179, + 113, 131, 0, 0, 0, 0, 143, 140, 0, 173, + 0, 176, 173, 0, 171, 155, 163, 0, 85, 0, + 200, 23, 0, 196, 151, 0, 0, 64, 65, 0, + 0, 0, 0, 0, 78, 62, 0, 0, 0, 0, + 93, 95, 0, 0, 99, 114, 132, 0, 134, 0, + 0, 141, 0, 0, 15, 0, 175, 177, 16, 170, + 0, 163, 18, 0, 0, 205, 24, 153, 0, 56, + 59, 66, 0, 68, 0, 70, 71, 72, 57, 0, + 0, 0, 63, 58, 74, 109, 0, 96, 133, 0, + 137, 144, 0, 0, 0, 172, 17, 164, 156, 157, + 160, 21, 155, 0, 0, 0, 67, 69, 0, 0, + 0, 97, 101, 142, 0, 178, 0, 0, 159, 161, + 162, 163, 154, 152, 60, 0, 0, 0, 0, 165, + 158, 166, 0, 82, 0, 0, 174, 13, 0, 0, + 79, 0, 80, 81, 167, 0, 83, 0, 168, } var yyTok1 = [...]int{ 1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 3, 3, 3, 3, 3, 3, 3, 77, 72, 3, - 48, 105, 75, 73, 56, 74, 83, 76, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 76, 71, 3, + 47, 104, 74, 72, 55, 73, 82, 75, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 50, 49, 51, 3, 3, 3, 3, 3, 3, 3, + 49, 48, 50, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 3, 3, 3, 3, 80, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 79, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 3, 3, 3, 3, 71, 3, 81, + 3, 3, 3, 3, 70, 3, 80, } var yyTok2 = [...]int{ @@ -572,11 +569,11 @@ var yyTok2 = [...]int{ 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, - 42, 43, 44, 45, 46, 47, 52, 53, 54, 55, + 42, 43, 44, 45, 46, 51, 52, 53, 54, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, - 67, 68, 69, 70, 78, 79, 82, 84, 85, 86, + 67, 68, 69, 77, 78, 81, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, - 97, 98, 99, 100, 101, 102, 103, 104, + 97, 98, 99, 100, 101, 102, 103, } var yyTok3 = [...]int{ 0, @@ -1695,151 +1692,151 @@ yydefault: yyVAL.str = "if" } case 137: - yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:762 - { - yyVAL.str = "values" - } - case 138: yyDollar = yyS[yypt-5 : yypt+1] - //line sql.y:768 + //line sql.y:764 { yyVAL.caseExpr = &CaseExpr{Expr: yyDollar[2].valExpr, Whens: yyDollar[3].whens, Else: yyDollar[4].valExpr} } - case 139: + case 138: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:773 + //line sql.y:769 { yyVAL.valExpr = nil } - case 140: + case 139: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:777 + //line sql.y:773 { yyVAL.valExpr = yyDollar[1].valExpr } - case 141: + case 140: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:783 + //line sql.y:779 { yyVAL.whens = []*When{yyDollar[1].when} } - case 142: + case 141: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:787 + //line sql.y:783 { yyVAL.whens = append(yyDollar[1].whens, yyDollar[2].when) } - case 143: + case 142: yyDollar = yyS[yypt-4 : yypt+1] - //line sql.y:793 + //line sql.y:789 { yyVAL.when = &When{Cond: yyDollar[2].boolExpr, Val: yyDollar[4].valExpr} } - case 144: + case 143: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:798 + //line sql.y:794 { yyVAL.valExpr = nil } - case 145: + case 144: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:802 + //line sql.y:798 { yyVAL.valExpr = yyDollar[2].valExpr } - case 146: + case 145: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:808 + //line sql.y:804 { yyVAL.colName = &ColName{Name: yyDollar[1].sqlID} } - case 147: + case 146: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:812 + //line sql.y:808 { yyVAL.colName = &ColName{Qualifier: yyDollar[1].sqlID, Name: yyDollar[3].sqlID} } + case 147: + yyDollar = yyS[yypt-1 : yypt+1] + //line sql.y:814 + { + yyVAL.valExpr = StrVal(yyDollar[1].bytes) + } case 148: yyDollar = yyS[yypt-1 : yypt+1] //line sql.y:818 { - yyVAL.valExpr = StrVal(yyDollar[1].bytes) + yyVAL.valExpr = NumVal(yyDollar[1].bytes) } case 149: yyDollar = yyS[yypt-1 : yypt+1] //line sql.y:822 { - yyVAL.valExpr = NumVal(yyDollar[1].bytes) + yyVAL.valExpr = ValArg(yyDollar[1].bytes) } case 150: yyDollar = yyS[yypt-1 : yypt+1] //line sql.y:826 - { - yyVAL.valExpr = ValArg(yyDollar[1].bytes) - } - case 151: - yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:830 { yyVAL.valExpr = &NullVal{} } - case 152: + case 151: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:835 + //line sql.y:831 { yyVAL.valExprs = nil } - case 153: + case 152: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:839 + //line sql.y:835 { yyVAL.valExprs = yyDollar[3].valExprs } - case 154: + case 153: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:844 + //line sql.y:840 { yyVAL.boolExpr = nil } - case 155: + case 154: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:848 + //line sql.y:844 { yyVAL.boolExpr = yyDollar[2].boolExpr } - case 156: + case 155: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:853 + //line sql.y:849 { yyVAL.orderBy = nil } - case 157: + case 156: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:857 + //line sql.y:853 { yyVAL.orderBy = yyDollar[3].orderBy } - case 158: + case 157: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:863 + //line sql.y:859 { yyVAL.orderBy = OrderBy{yyDollar[1].order} } - case 159: + case 158: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:867 + //line sql.y:863 { yyVAL.orderBy = append(yyDollar[1].orderBy, yyDollar[3].order) } - case 160: + case 159: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:873 + //line sql.y:869 { yyVAL.order = &Order{Expr: yyDollar[1].valExpr, Direction: yyDollar[2].str} } - case 161: + case 160: yyDollar = yyS[yypt-0 : yypt+1] + //line sql.y:874 + { + yyVAL.str = AST_ASC + } + case 161: + yyDollar = yyS[yypt-1 : yypt+1] //line sql.y:878 { yyVAL.str = AST_ASC @@ -1847,48 +1844,42 @@ yydefault: case 162: yyDollar = yyS[yypt-1 : yypt+1] //line sql.y:882 - { - yyVAL.str = AST_ASC - } - case 163: - yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:886 { yyVAL.str = AST_DESC } - case 164: + case 163: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:891 + //line sql.y:887 { yyVAL.limit = nil } - case 165: + case 164: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:895 + //line sql.y:891 { yyVAL.limit = &Limit{Rowcount: yyDollar[2].valExpr} } - case 166: + case 165: yyDollar = yyS[yypt-4 : yypt+1] - //line sql.y:899 + //line sql.y:895 { yyVAL.limit = &Limit{Offset: yyDollar[2].valExpr, Rowcount: yyDollar[4].valExpr} } - case 167: + case 166: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:904 + //line sql.y:900 { yyVAL.str = "" } - case 168: + case 167: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:908 + //line sql.y:904 { yyVAL.str = AST_FOR_UPDATE } - case 169: + case 168: yyDollar = yyS[yypt-4 : yypt+1] - //line sql.y:912 + //line sql.y:908 { if yyDollar[3].sqlID != "share" { yylex.Error("expecting share") @@ -1900,123 +1891,129 @@ yydefault: } yyVAL.str = AST_SHARE_MODE } - case 170: + case 169: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:925 + //line sql.y:921 { yyVAL.columns = nil } - case 171: + case 170: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:929 + //line sql.y:925 { yyVAL.columns = yyDollar[2].columns } - case 172: + case 171: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:935 + //line sql.y:931 { yyVAL.columns = Columns{&NonStarExpr{Expr: yyDollar[1].colName}} } - case 173: + case 172: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:939 + //line sql.y:935 { yyVAL.columns = append(yyVAL.columns, &NonStarExpr{Expr: yyDollar[3].colName}) } - case 174: + case 173: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:944 + //line sql.y:940 { yyVAL.updateExprs = nil } - case 175: + case 174: yyDollar = yyS[yypt-5 : yypt+1] - //line sql.y:948 + //line sql.y:944 { yyVAL.updateExprs = yyDollar[5].updateExprs } - case 176: + case 175: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:954 + //line sql.y:950 { yyVAL.insRows = yyDollar[2].values } - case 177: + case 176: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:958 + //line sql.y:954 { yyVAL.insRows = yyDollar[1].selStmt } - case 178: + case 177: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:964 + //line sql.y:960 { yyVAL.values = Values{yyDollar[1].rowTuple} } - case 179: + case 178: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:968 + //line sql.y:964 { yyVAL.values = append(yyDollar[1].values, yyDollar[3].rowTuple) } - case 180: + case 179: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:974 + //line sql.y:970 { yyVAL.rowTuple = ValTuple(yyDollar[2].valExprs) } - case 181: + case 180: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:978 + //line sql.y:974 { yyVAL.rowTuple = yyDollar[1].subquery } - case 182: + case 181: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:984 + //line sql.y:980 { yyVAL.updateExprs = UpdateExprs{yyDollar[1].updateExpr} } - case 183: + case 182: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:988 + //line sql.y:984 { yyVAL.updateExprs = append(yyDollar[1].updateExprs, yyDollar[3].updateExpr) } - case 184: + case 183: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:994 + //line sql.y:990 { yyVAL.updateExpr = &UpdateExpr{Name: yyDollar[1].colName, Expr: yyDollar[3].valExpr} } - case 185: + case 184: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:999 + //line sql.y:995 + { + yyVAL.empty = struct{}{} + } + case 185: + yyDollar = yyS[yypt-2 : yypt+1] + //line sql.y:997 { yyVAL.empty = struct{}{} } case 186: - yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:1001 + yyDollar = yyS[yypt-0 : yypt+1] + //line sql.y:1000 { yyVAL.empty = struct{}{} } case 187: - yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:1004 + yyDollar = yyS[yypt-3 : yypt+1] + //line sql.y:1002 { yyVAL.empty = struct{}{} } case 188: - yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:1006 + yyDollar = yyS[yypt-0 : yypt+1] + //line sql.y:1005 { yyVAL.empty = struct{}{} } case 189: - yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:1009 + yyDollar = yyS[yypt-1 : yypt+1] + //line sql.y:1007 { yyVAL.empty = struct{}{} } @@ -2028,100 +2025,94 @@ yydefault: } case 191: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:1015 + //line sql.y:1013 { yyVAL.empty = struct{}{} } case 192: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:1017 + //line sql.y:1015 { yyVAL.empty = struct{}{} } case 193: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:1019 + //line sql.y:1017 { yyVAL.empty = struct{}{} } case 194: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:1021 + //line sql.y:1019 { yyVAL.empty = struct{}{} } case 195: - yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:1023 + yyDollar = yyS[yypt-0 : yypt+1] + //line sql.y:1022 { yyVAL.empty = struct{}{} } case 196: - yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:1026 + yyDollar = yyS[yypt-1 : yypt+1] + //line sql.y:1024 { yyVAL.empty = struct{}{} } case 197: - yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:1028 + yyDollar = yyS[yypt-0 : yypt+1] + //line sql.y:1027 { yyVAL.empty = struct{}{} } case 198: - yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:1031 + yyDollar = yyS[yypt-1 : yypt+1] + //line sql.y:1029 { yyVAL.empty = struct{}{} } case 199: - yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:1033 + yyDollar = yyS[yypt-0 : yypt+1] + //line sql.y:1032 { yyVAL.empty = struct{}{} } case 200: - yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:1036 + yyDollar = yyS[yypt-2 : yypt+1] + //line sql.y:1034 { yyVAL.empty = struct{}{} } case 201: - yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:1038 - { - yyVAL.empty = struct{}{} - } - case 202: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:1042 + //line sql.y:1038 { yyVAL.sqlID = SQLName(strings.ToLower(string(yyDollar[1].bytes))) } - case 203: + case 202: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:1048 + //line sql.y:1044 { yyVAL.sqlID = SQLName(yyDollar[1].bytes) } - case 204: + case 203: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:1054 + //line sql.y:1050 { if incNesting(yylex) { yylex.Error("max nesting level reached") return 1 } } - case 205: + case 204: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:1063 + //line sql.y:1059 { decNesting(yylex) } - case 206: + case 205: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:1068 + //line sql.y:1064 { forceEOF(yylex) } diff --git a/go/vt/sqlparser/sql.y b/go/vt/sqlparser/sql.y index 500afb5e2e..14f7393f85 100644 --- a/go/vt/sqlparser/sql.y +++ b/go/vt/sqlparser/sql.y @@ -73,8 +73,8 @@ func forceEOF(yylex interface{}) { %token LEX_ERROR %token SELECT INSERT UPDATE DELETE FROM WHERE GROUP HAVING ORDER BY LIMIT FOR -%token ALL DISTINCT AS EXISTS IN IS LIKE BETWEEN NULL ASC DESC VALUES INTO DUPLICATE KEY DEFAULT SET LOCK KEYRANGE -%token VALUE, LAST_INSERT_ID +%token ALL DISTINCT AS EXISTS IN IS LIKE BETWEEN NULL ASC DESC INTO DUPLICATE KEY DEFAULT SET LOCK KEYRANGE +%token VALUES LAST_INSERT_ID %token ID STRING NUMBER VALUE_ARG LIST_ARG COMMENT %token LE GE NE NULL_SAFE_EQUAL %token '(' '=' '<' '>' @@ -758,10 +758,6 @@ keyword_as_func: { $$ = "if" } -| VALUES - { - $$ = "values" - } case_expression: CASE value_expression_opt when_expression_list else_expression_opt END diff --git a/go/vt/sqlparser/token.go b/go/vt/sqlparser/token.go index 808da42599..a73dc5912a 100644 --- a/go/vt/sqlparser/token.go +++ b/go/vt/sqlparser/token.go @@ -104,7 +104,6 @@ var keywords = map[string]int{ "use": USE, "using": USING, "values": VALUES, - "value": VALUE, "view": VIEW, "when": WHEN, "where": WHERE, diff --git a/test/queryservice_tests/nocache_tests.py b/test/queryservice_tests/nocache_tests.py index 26cd6bbca4..f76b6ec852 100644 --- a/test/queryservice_tests/nocache_tests.py +++ b/test/queryservice_tests/nocache_tests.py @@ -163,12 +163,6 @@ class TestNocache(framework.TestCase): with self.assertRaises(dbexceptions.DatabaseError): self.env.execute("insert into upsert_test(id1, id2) values (1, 1)") self.env.execute("insert into upsert_test(id1, id2) values (2, 1) on duplicate key update id2 = 2") - - # TODO(sougou): add test for values clause after bug fix on handling - # values. - self.env.execute("delete from upsert_test") - with self.assertRaises(dbexceptions.DatabaseError): - self.env.execute("insert into upsert_test(id1, id2) values (1, 1) on duplicate key update id2 = last_insert_id(id1)") finally: self.env.conn.rollback() From c9945d29e51f572be47ac39eb6a042d753232f30 Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Sat, 15 Aug 2015 18:34:47 -0700 Subject: [PATCH 05/80] vtgate v3: Add TODO for handling upserts --- go/vt/vtgate/router.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/go/vt/vtgate/router.go b/go/vt/vtgate/router.go index 30ae523d2b..3d83b0749a 100644 --- a/go/vt/vtgate/router.go +++ b/go/vt/vtgate/router.go @@ -481,6 +481,9 @@ func (rtr *Router) handlePrimary(vcursor *requestContext, vindexKey interface{}, return "", 0, err } } else { + // TODO(sougou): I think we have to ignore dup key error if this was + // an upsert. For now, I'm punting on this because this would be a very + // uncommon use case. We should revisit this when work on v3 resumes. err = colVindex.Vindex.(planbuilder.Functional).Create(vcursor, vindexKey) if err != nil { return "", 0, err From f8fbf3d891c7110bb2effc88269c8a5cfef4120a Mon Sep 17 00:00:00 2001 From: Michael Berlin Date: Sun, 16 Aug 2015 23:19:09 -0700 Subject: [PATCH 06/80] proto: Fix typos in query.proto. --- go/vt/proto/query/query.pb.go | 4 ++-- proto/query.proto | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go/vt/proto/query/query.pb.go b/go/vt/proto/query/query.pb.go index 5b39b60ba3..abc0717206 100644 --- a/go/vt/proto/query/query.pb.go +++ b/go/vt/proto/query/query.pb.go @@ -264,7 +264,7 @@ func (m *Target) String() string { return proto.CompactTextString(m) } func (*Target) ProtoMessage() {} // VTGateCallerID is sent by VTGate to VTTablet to describe the -// caller. If possible, this enformation is secure. For instance, +// caller. If possible, this information is secure. For instance, // if using unique certificates that guarantee that VTGate->VTTablet // traffic cannot be spoofed, then VTTablet can trust this information, // and VTTablet will use it for tablet ACLs, for instance. @@ -799,7 +799,7 @@ type RealtimeStats struct { // we do not send queries to servers that are not healthy. HealthError string `protobuf:"bytes,1,opt,name=health_error" json:"health_error,omitempty"` // seconds_behind_master is populated for slaves only. It indicates - // how far nehind on replication a slave currently is. It is used + // how far behind on replication a slave currently is. It is used // by clients for subset selection (so we don't try to send traffic // to tablets that are too far behind). SecondsBehindMaster uint32 `protobuf:"varint,2,opt,name=seconds_behind_master" json:"seconds_behind_master,omitempty"` diff --git a/proto/query.proto b/proto/query.proto index 3a6a89f614..73552eb923 100644 --- a/proto/query.proto +++ b/proto/query.proto @@ -19,7 +19,7 @@ message Target { } // VTGateCallerID is sent by VTGate to VTTablet to describe the -// caller. If possible, this enformation is secure. For instance, +// caller. If possible, this information is secure. For instance, // if using unique certificates that guarantee that VTGate->VTTablet // traffic cannot be spoofed, then VTTablet can trust this information, // and VTTablet will use it for tablet ACLs, for instance. @@ -282,7 +282,7 @@ message RealtimeStats { string health_error = 1; // seconds_behind_master is populated for slaves only. It indicates - // how far nehind on replication a slave currently is. It is used + // how far behind on replication a slave currently is. It is used // by clients for subset selection (so we don't try to send traffic // to tablets that are too far behind). uint32 seconds_behind_master = 2; From 99ad60913801e29dfb55337eaf310e407457010c Mon Sep 17 00:00:00 2001 From: Michael Berlin Date: Sun, 16 Aug 2015 23:19:44 -0700 Subject: [PATCH 07/80] wrangler: Add error prefix to VtctlPipe. --- go/vt/wrangler/testlib/vtctl_pipe.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/go/vt/wrangler/testlib/vtctl_pipe.go b/go/vt/wrangler/testlib/vtctl_pipe.go index fe4725a170..42f1e03795 100644 --- a/go/vt/wrangler/testlib/vtctl_pipe.go +++ b/go/vt/wrangler/testlib/vtctl_pipe.go @@ -5,6 +5,7 @@ package testlib import ( + "fmt" "net" "testing" "time" @@ -75,7 +76,7 @@ func (vp *VtctlPipe) Run(args []string) error { c, errFunc, err := vp.client.ExecuteVtctlCommand(ctx, args, actionTimeout, lockTimeout) if err != nil { - return err + return fmt.Errorf("VtctlPipe.Run() failed: %v", err) } for le := range c { vp.t.Logf(le.String()) From 49e5fadd07ad1f4adb1f9dc893858b746040eba7 Mon Sep 17 00:00:00 2001 From: Michael Berlin Date: Sun, 16 Aug 2015 23:25:09 -0700 Subject: [PATCH 08/80] tabletserver: Add TODO where it's not clear if a channel should be closed to prevent resource exhaustion. --- go/vt/tabletserver/grpcqueryservice/server.go | 1 + 1 file changed, 1 insertion(+) diff --git a/go/vt/tabletserver/grpcqueryservice/server.go b/go/vt/tabletserver/grpcqueryservice/server.go index 1f97fa7aa1..3c5debf84d 100644 --- a/go/vt/tabletserver/grpcqueryservice/server.go +++ b/go/vt/tabletserver/grpcqueryservice/server.go @@ -194,6 +194,7 @@ func (q *query) StreamHealth(request *pb.StreamHealthRequest, stream pbs.Query_S for shr := range c { // we send until the client disconnects if err := stream.Send(shr); err != nil { + // TODO(mberlin): Shouldn't we explicitly close c here? return } } From 7a572c9bb46cf52a5f7398be5b7268d0f64848e3 Mon Sep 17 00:00:00 2001 From: Michael Berlin Date: Sun, 16 Aug 2015 23:25:42 -0700 Subject: [PATCH 09/80] proto: Add filtered_replication_synced_until_timestamp to RealtimeStats. --- go/vt/proto/query/query.pb.go | 8 +++++++- proto/query.proto | 11 ++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/go/vt/proto/query/query.pb.go b/go/vt/proto/query/query.pb.go index abc0717206..3be9a634e5 100644 --- a/go/vt/proto/query/query.pb.go +++ b/go/vt/proto/query/query.pb.go @@ -799,10 +799,16 @@ type RealtimeStats struct { // we do not send queries to servers that are not healthy. HealthError string `protobuf:"bytes,1,opt,name=health_error" json:"health_error,omitempty"` // seconds_behind_master is populated for slaves only. It indicates - // how far behind on replication a slave currently is. It is used + // how far behind on (MySQL) replication a slave currently is. It is used // by clients for subset selection (so we don't try to send traffic // to tablets that are too far behind). + // TODO(mberlin): Let's switch it to int64 instead? SecondsBehindMaster uint32 `protobuf:"varint,2,opt,name=seconds_behind_master" json:"seconds_behind_master,omitempty"` + // filtered_replication_synced_until_timestamp is populated for the receiving + // master of an ongoing filtered replication only. + // It is used to find out how far the receiving master lags behind the + // source shard. + FilteredReplicationSyncedUntilTimestamp int64 `protobuf:"varint,4,opt,name=filtered_replication_synced_until_timestamp" json:"filtered_replication_synced_until_timestamp,omitempty"` // cpu_usage is used for load-based balancing CpuUsage float64 `protobuf:"fixed64,3,opt,name=cpu_usage" json:"cpu_usage,omitempty"` } diff --git a/proto/query.proto b/proto/query.proto index 73552eb923..42591d647d 100644 --- a/proto/query.proto +++ b/proto/query.proto @@ -281,12 +281,21 @@ message RealtimeStats { // we do not send queries to servers that are not healthy. string health_error = 1; + // TODO(mberlin): Add an health error for filtered replication? + // seconds_behind_master is populated for slaves only. It indicates - // how far behind on replication a slave currently is. It is used + // how far behind on (MySQL) replication a slave currently is. It is used // by clients for subset selection (so we don't try to send traffic // to tablets that are too far behind). + // TODO(mberlin): Let's switch it to int64 instead? uint32 seconds_behind_master = 2; + // filtered_replication_synced_until_timestamp is populated for the receiving + // master of an ongoing filtered replication only. + // It is used to find out how far the receiving master lags behind the + // source shard. + int64 filtered_replication_synced_until_timestamp = 4; + // cpu_usage is used for load-based balancing double cpu_usage = 3; } From a281f605a6f8164660450b68b7670f52689b46de Mon Sep 17 00:00:00 2001 From: Michael Berlin Date: Mon, 17 Aug 2015 00:17:25 -0700 Subject: [PATCH 10/80] vtctl: Add command WaitForFilteredReplication. --- go/vt/tabletserver/sqlquery.go | 5 + go/vt/vtctl/vtctl.go | 92 ++++++++- .../wait_for_filtered_replication_test.go | 178 ++++++++++++++++++ 3 files changed, 272 insertions(+), 3 deletions(-) create mode 100644 go/vt/wrangler/testlib/wait_for_filtered_replication_test.go diff --git a/go/vt/tabletserver/sqlquery.go b/go/vt/tabletserver/sqlquery.go index df4b876d53..f0cb08751d 100644 --- a/go/vt/tabletserver/sqlquery.go +++ b/go/vt/tabletserver/sqlquery.go @@ -119,6 +119,11 @@ func NewSqlQuery(config Config) *SqlQuery { return sq } +// SetTargetForTest allows to modify the target without calling allowQueries. +func (sq *SqlQuery) SetTargetForTest(target *pb.Target) { + sq.target = target +} + // GetState returns the name of the current SqlQuery state. func (sq *SqlQuery) GetState() string { sq.mu.Lock() diff --git a/go/vt/vtctl/vtctl.go b/go/vt/vtctl/vtctl.go index e01e1412a7..f0f5c7ba31 100644 --- a/go/vt/vtctl/vtctl.go +++ b/go/vt/vtctl/vtctl.go @@ -95,19 +95,20 @@ import ( "time" log "github.com/golang/glog" + "golang.org/x/net/context" + "github.com/youtube/vitess/go/flagutil" "github.com/youtube/vitess/go/jscfg" "github.com/youtube/vitess/go/netutil" hk "github.com/youtube/vitess/go/vt/hook" "github.com/youtube/vitess/go/vt/logutil" myproto "github.com/youtube/vitess/go/vt/mysqlctl/proto" + pb "github.com/youtube/vitess/go/vt/proto/topodata" "github.com/youtube/vitess/go/vt/tabletmanager/actionnode" + "github.com/youtube/vitess/go/vt/tabletserver/tabletconn" "github.com/youtube/vitess/go/vt/topo" "github.com/youtube/vitess/go/vt/topotools" "github.com/youtube/vitess/go/vt/wrangler" - "golang.org/x/net/context" - - pb "github.com/youtube/vitess/go/vt/proto/topodata" ) var ( @@ -234,6 +235,9 @@ var commands = []commandGroup{ command{"ShardReplicationFix", commandShardReplicationFix, " ", "Walks through a ShardReplication object and fixes the first error that it encounters."}, + command{"WaitForFilteredReplication", commandWaitForFilteredReplication, + "[-max_delay ] ", + "Blocks until the specified shard has caught up with the filtered replication of its source shard."}, command{"RemoveShardCell", commandRemoveShardCell, "[-force] [-recursive] ", "Removes the cell from the shard's Cells list."}, @@ -1330,6 +1334,88 @@ func commandShardReplicationFix(ctx context.Context, wr *wrangler.Wrangler, subF return topo.FixShardReplication(ctx, wr.TopoServer(), wr.Logger(), cell, keyspace, shard) } +func commandWaitForFilteredReplication(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { + // Allowed tolerance up to which the local clock may run ahead. Otherwise, the command will fail. + const allowedAheadTime = time.Second + maxDelay := subFlags.Duration("max_delay", 30*time.Second, + "Specifies the maximum delay, in seconds, the filtered replication of the"+ + " given destination shard should lag behind the source shard. When"+ + " higher, the command will block and wait for the delay to decrease.") + if err := subFlags.Parse(args); err != nil { + return err + } + if subFlags.NArg() != 1 { + return fmt.Errorf("The argument is required for the WaitForFilteredReplication command.") + } + keyspace, shard, err := topo.ParseKeyspaceShardString(subFlags.Arg(0)) + if err != nil { + return err + } + shardInfo, err := wr.TopoServer().GetShard(ctx, keyspace, shard) + if err != nil { + return err + } + if len(shardInfo.SourceShards) == 0 { + return fmt.Errorf("shard %v/%v has no source shard", keyspace, shard) + } + alias := shardInfo.MasterAlias + if topo.TabletAliasIsZero(alias) { + return fmt.Errorf("shard %v/%v has no master", keyspace, shard) + } + tabletInfo, err := wr.TopoServer().GetTablet(ctx, alias) + if err != nil { + return err + } + ep, err := topo.TabletEndPoint(tabletInfo.Tablet) + if err != nil { + return fmt.Errorf("cannot get EndPoint for master tablet record: %v record: %v", err, tabletInfo) + } + + // pass in a non-UNKNOWN tablet type to not use sessionId + conn, err := tabletconn.GetDialer()(ctx, ep, "", "", pb.TabletType_MASTER, 30*time.Second) + if err != nil { + return fmt.Errorf("cannot connect to tablet %v: %v", alias, err) + } + + stream, errFunc, err := conn.StreamHealth(ctx) + if err != nil { + return fmt.Errorf("could not stream health records from tablet: %v err: %v", alias, err) + } + var lastSeenDelay int + for { + select { + case <-ctx.Done(): + return fmt.Errorf("context was done before filtered replication did caught up. Last seen delay: %v context Error: %v", lastSeenDelay, ctx.Err()) + case shr, ok := <-stream: + if !ok { + return fmt.Errorf("stream ended early: %v", errFunc()) + } + gotTarget := shr.GetTarget() + if gotTarget == nil { + return fmt.Errorf("stream health record did not include Target: %v", shr) + } + if gotTarget.Keyspace != keyspace || gotTarget.Shard != shard { + return fmt.Errorf("received health record for wrong tablet. Expected tablet: %v/%v received health record: %v", keyspace, shard, shr) + } + if gotTarget.TabletType != pb.TabletType_MASTER { + return fmt.Errorf("tablet: %v should be master, but is not. type: %v", alias, gotTarget.TabletType.String()) + } + + now := time.Now() + lastSync := time.Unix(shr.GetRealtimeStats().FilteredReplicationSyncedUntilTimestamp, 0 /* nsecs */) + lastSeenDelay := now.Sub(lastSync) + if lastSeenDelay < -allowedAheadTime { + return fmt.Errorf("cannot reliably wait for the filtered replication to catch up. The local clock (runs behind) or the tablet clock (runs ahead) is not synchronized. seen delay: %v local clock now: %v last sync on tablet (%v): %v", lastSeenDelay, now, alias, lastSync) + } + if lastSeenDelay <= *maxDelay { + wr.Logger().Printf("Filtered replication on tablet: %v has caught up. Last seen delay: %.1f seconds\n", alias, lastSeenDelay.Seconds()) + return nil + } + wr.Logger().Printf("Waiting for filtered replication to catch up on tablet: %v Last seen delay: %.1f seconds\n", alias, lastSeenDelay.Seconds()) + } + } +} + func commandRemoveShardCell(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { force := subFlags.Bool("force", false, "Proceeds even if the cell's topology server cannot be reached. The assumption is that you turned down the entire cell, and just need to update the global topo data.") recursive := subFlags.Bool("recursive", false, "Also delete all tablets in that cell belonging to the specified shard.") diff --git a/go/vt/wrangler/testlib/wait_for_filtered_replication_test.go b/go/vt/wrangler/testlib/wait_for_filtered_replication_test.go new file mode 100644 index 0000000000..8b055d64f9 --- /dev/null +++ b/go/vt/wrangler/testlib/wait_for_filtered_replication_test.go @@ -0,0 +1,178 @@ +// Copyright 2015, Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package testlib + +import ( + "fmt" + "math/rand" + "strings" + "testing" + "time" + + "golang.org/x/net/context" + + "github.com/youtube/vitess/go/vt/logutil" + "github.com/youtube/vitess/go/vt/tabletmanager/tmclient" + "github.com/youtube/vitess/go/vt/tabletserver" + "github.com/youtube/vitess/go/vt/tabletserver/grpcqueryservice" + "github.com/youtube/vitess/go/vt/wrangler" + "github.com/youtube/vitess/go/vt/zktopo" + + pbq "github.com/youtube/vitess/go/vt/proto/query" + pbt "github.com/youtube/vitess/go/vt/proto/topodata" +) + +const keyspace = "ks" +const destShard = "-80" + +// TestWaitForFilteredReplication tests the vtctl command "WaitForFilteredReplication". +// WaitForFilteredReplication ensures that the dest shard has caught up +// with the source shard up to a maximum replication delay (in seconds). +func TestWaitForFilteredReplication(t *testing.T) { + target := &pbq.Target{Keyspace: keyspace, Shard: destShard, TabletType: pbt.TabletType_MASTER} + + waitForFilteredReplicationDefaultDelay(t, target, "" /* expectedErr */) +} + +// TestWaitForFilteredReplication_nonMasterFails tests that +// vtctl WaitForFilteredReplication fails if the queried tablet is not MASTER. +func TestWaitForFilteredReplication_nonMasterFails(t *testing.T) { + target := &pbq.Target{Keyspace: keyspace, Shard: destShard, TabletType: pbt.TabletType_REPLICA} + + waitForFilteredReplicationDefaultDelay(t, target, "should be master, but is not") +} + +// TestWaitForFilteredReplication_wrongTarget tests that +// vtctl WaitForFilteredReplication fails if the target is different than expected. +func TestWaitForFilteredReplication_wrongTarget(t *testing.T) { + target := &pbq.Target{Keyspace: keyspace, Shard: "wrongshard", TabletType: pbt.TabletType_MASTER} + + waitForFilteredReplicationDefaultDelay(t, target, "received health record for wrong tablet") +} + +// TestWaitForFilteredReplication_unsyncClocks tests that +// vtctl WaitForFilteredReplication fails if the calculated delay is negative. +func TestWaitForFilteredReplication_unsyncClocks(t *testing.T) { + target := &pbq.Target{Keyspace: keyspace, Shard: destShard, TabletType: pbt.TabletType_MASTER} + + // Replication is lagging behind. + oneHourDelay := &pbq.RealtimeStats{ + FilteredReplicationSyncedUntilTimestamp: time.Now().Add(-time.Hour).Unix(), + } + + // Tablet is one hour ahead of the local clock. + negativeDelayFunc := func() *pbq.RealtimeStats { + return &pbq.RealtimeStats{ + FilteredReplicationSyncedUntilTimestamp: time.Now().Add(time.Hour).Unix(), + } + } + + waitForFilteredReplication(t, target, "cannot reliably wait for the filtered replication to catch up", oneHourDelay, negativeDelayFunc) +} + +// TestWaitForFilteredReplication_unsyncClocksTolerance tests that +// vtctl WaitForFilteredReplication succeeds as long as the calculated +// negative delay is above a certain tolerance. +func TestWaitForFilteredReplication_unsyncClocksTolerance(t *testing.T) { + target := &pbq.Target{Keyspace: keyspace, Shard: destShard, TabletType: pbt.TabletType_MASTER} + + // Replication is lagging behind. + oneHourDelay := &pbq.RealtimeStats{ + FilteredReplicationSyncedUntilTimestamp: time.Now().Add(-time.Hour).Unix(), + } + + // Tablet is half a second ahead of the local clock. + slightNegativeDelayFunc := func() *pbq.RealtimeStats { + return &pbq.RealtimeStats{ + FilteredReplicationSyncedUntilTimestamp: time.Now().Add(time.Second / 2).Unix(), + } + } + + waitForFilteredReplication(t, target, "", oneHourDelay, slightNegativeDelayFunc) +} + +func waitForFilteredReplicationDefaultDelay(t *testing.T, target *pbq.Target, expectedErr string) { + // Replication is lagging behind. + oneHourDelay := &pbq.RealtimeStats{ + FilteredReplicationSyncedUntilTimestamp: time.Now().Add(-time.Hour).Unix(), + } + + // Replication caught up. + oneSecondDelayFunc := func() *pbq.RealtimeStats { + return &pbq.RealtimeStats{ + FilteredReplicationSyncedUntilTimestamp: time.Now().Add(-time.Second).Unix(), + } + } + + waitForFilteredReplication(t, target, expectedErr, oneHourDelay, oneSecondDelayFunc) +} + +func waitForFilteredReplication(t *testing.T, target *pbq.Target, expectedErr string, initialStats *pbq.RealtimeStats, broadcastStatsFunc func() *pbq.RealtimeStats) { + ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"}) + wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), time.Second) + vp := NewVtctlPipe(t, ts) + defer vp.Close() + + // source of the filtered replication. We don't start its loop because we don't connect to it. + source := NewFakeTablet(t, wr, "cell1", 0, pbt.TabletType_MASTER, + TabletKeyspaceShard(t, keyspace, "0")) + // dest is the master of the dest shard which receives filtered replication events. + dest := NewFakeTablet(t, wr, "cell1", 1, pbt.TabletType_MASTER, + TabletKeyspaceShard(t, keyspace, destShard)) + dest.StartActionLoop(t, wr) + defer dest.StopActionLoop(t) + + // Build topology state as we would expect it when filtered replication is enabled. + ctx := context.Background() + wr.SetSourceShards(ctx, keyspace, destShard, []*pbt.TabletAlias{source.Tablet.GetAlias()}, nil) + + // Use real, but trimmed down QueryService. + testConfig := tabletserver.DefaultQsConfig + testConfig.EnablePublishStats = false + testConfig.DebugURLPrefix = fmt.Sprintf("TestWaitForFilteredReplication-%d-", rand.Int63()) + qs := tabletserver.NewSqlQuery(testConfig) + qs.SetTargetForTest(target) + grpcqueryservice.RegisterForTest(dest.RPCServer, qs) + + qs.BroadcastHealth(42, initialStats) + + // run vtctl WaitForFilteredReplication + stopBroadcasting := make(chan struct{}) + go func() { + defer close(stopBroadcasting) + err := vp.Run([]string{"WaitForFilteredReplication", "-max_delay", "10s", dest.Tablet.Keyspace + "/" + dest.Tablet.Shard}) + if expectedErr == "" { + if err != nil { + t.Fatalf("WaitForFilteredReplication must not fail: %v", err) + } + } else { + if err == nil || !strings.Contains(err.Error(), expectedErr) { + t.Fatalf("WaitForFilteredReplication wrong error. got: %v want substring: %v", err, expectedErr) + } + } + }() + + // Broadcast health record as long as vtctl is running. + for { + // Give vtctl a head start to consume the initial stats. + // (We do this because there's unfortunately no way to explicitly + // synchronize with the point where conn.StreamHealth() has started.) + // (Tests won't break if vtctl misses the initial stats. Only coverage + // will be impacted.) + timer := time.NewTimer(1 * time.Millisecond) + + select { + case <-stopBroadcasting: + timer.Stop() + return + case <-timer.C: + qs.BroadcastHealth(42, broadcastStatsFunc()) + // Pace the flooding broadcasting to waste less CPU. + timer.Reset(1 * time.Millisecond) + } + } + + // vtctl WaitForFilteredReplication returned. +} From e517d9bd5881a0e6e98aa152953bf43574853f9c Mon Sep 17 00:00:00 2001 From: Alain Jobart Date: Mon, 17 Aug 2015 07:15:18 -0700 Subject: [PATCH 11/80] Removing two jscfg functions. --- go/jscfg/jscfg.go | 29 +---------------------------- go/vt/dbconfigs/credentials.go | 12 ++++++++++-- go/vt/tabletmanager/agent.go | 15 ++++++++++----- 3 files changed, 21 insertions(+), 35 deletions(-) diff --git a/go/jscfg/jscfg.go b/go/jscfg/jscfg.go index 1662fb43db..5336f9e188 100644 --- a/go/jscfg/jscfg.go +++ b/go/jscfg/jscfg.go @@ -5,13 +5,7 @@ // Package jscfg implements a simple API for reading and writing JSON files. package jscfg -import ( - "encoding/json" - "fmt" - "io/ioutil" - - "github.com/youtube/vitess/go/ioutil2" -) +import "encoding/json" // ToJSON converts a structure to JSON, or panics func ToJSON(val interface{}) string { @@ -23,24 +17,3 @@ func ToJSON(val interface{}) string { } return string(data) } - -// WriteJSON atomically write a marshaled structure to disk. -func WriteJSON(filename string, val interface{}) error { - data, err := json.MarshalIndent(val, " ", " ") - if err != nil { - return fmt.Errorf("WriteJSON failed: %v %v", filename, err) - } - return ioutil2.WriteFileAtomic(filename, data, 0660) -} - -// ReadJSON reads and unmarshals a JSON file -func ReadJSON(filename string, val interface{}) error { - data, err := ioutil.ReadFile(filename) - if err != nil { - return fmt.Errorf("ReadJSON failed: %T %v", val, err) - } - if err = json.Unmarshal(data, val); err != nil { - return fmt.Errorf("ReadJSON failed: %T %v %v", val, filename, err) - } - return nil -} diff --git a/go/vt/dbconfigs/credentials.go b/go/vt/dbconfigs/credentials.go index d336b7653b..1b1011f0a7 100644 --- a/go/vt/dbconfigs/credentials.go +++ b/go/vt/dbconfigs/credentials.go @@ -10,12 +10,13 @@ package dbconfigs // link with this library, so we should be safe. import ( + "encoding/json" "errors" "flag" + "io/ioutil" "sync" log "github.com/golang/glog" - "github.com/youtube/vitess/go/jscfg" ) var ( @@ -74,10 +75,17 @@ func (fcs *FileCredentialsServer) GetUserAndPassword(user string) (string, strin // read the json file only once if fcs.dbCredentials == nil { fcs.dbCredentials = make(map[string][]string) - if err := jscfg.ReadJSON(*dbCredentialsFile, &fcs.dbCredentials); err != nil { + + data, err := ioutil.ReadFile(*dbCredentialsFile) + if err != nil { log.Warningf("Failed to read dbCredentials file: %v", *dbCredentialsFile) return "", "", err } + + if err = json.Unmarshal(data, &fcs.dbCredentials); err != nil { + log.Warningf("Failed to parse dbCredentials file: %v", *dbCredentialsFile) + return "", "", err + } } passwd, ok := fcs.dbCredentials[user] diff --git a/go/vt/tabletmanager/agent.go b/go/vt/tabletmanager/agent.go index 3a718f0606..f549917928 100644 --- a/go/vt/tabletmanager/agent.go +++ b/go/vt/tabletmanager/agent.go @@ -25,6 +25,7 @@ import ( "encoding/json" "flag" "fmt" + "io/ioutil" "net" "sync" "time" @@ -34,7 +35,6 @@ import ( log "github.com/golang/glog" "github.com/youtube/vitess/go/event" "github.com/youtube/vitess/go/history" - "github.com/youtube/vitess/go/jscfg" "github.com/youtube/vitess/go/netutil" "github.com/youtube/vitess/go/stats" "github.com/youtube/vitess/go/trace" @@ -112,12 +112,17 @@ func loadSchemaOverrides(overridesFile string) []tabletserver.SchemaOverride { if overridesFile == "" { return schemaOverrides } - if err := jscfg.ReadJSON(overridesFile, &schemaOverrides); err != nil { + data, err := ioutil.ReadFile(overridesFile) + if err != nil { log.Warningf("can't read overridesFile %v: %v", overridesFile, err) - } else { - data, _ := json.MarshalIndent(schemaOverrides, "", " ") - log.Infof("schemaOverrides: %s\n", data) + return schemaOverrides } + if err = json.Unmarshal(data, &schemaOverrides); err != nil { + log.Warningf("can't parse overridesFile %v: %v", overridesFile, err) + return schemaOverrides + } + data, _ = json.MarshalIndent(schemaOverrides, "", " ") + log.Infof("schemaOverrides: %s\n", data) return schemaOverrides } From 9df8070ac245fa78eb62ebf1d865877c99a2c30c Mon Sep 17 00:00:00 2001 From: Alain Jobart Date: Mon, 17 Aug 2015 08:13:58 -0700 Subject: [PATCH 12/80] Removing jscfg package. It wasn't doing much, except using panics to return errors. --- go/jscfg/jscfg.go | 19 ---- go/vt/hook/hook.go | 27 ++++- go/vt/mysqlctl/proto/schema.go | 15 +-- go/vt/mysqlctl/proto/schema_test.go | 2 +- go/vt/tabletmanager/actionnode/actionnode.go | 25 +++-- .../tabletmanager/actionnode/structs_test.go | 20 ++-- go/vt/tabletmanager/actionnode/utils.go | 51 ++++++++- go/vt/vtctl/query.go | 13 +-- go/vt/vtctl/vtctl.go | 106 ++++++++++-------- go/vt/worker/split_clone.go | 3 +- go/vt/wrangler/zkns.go | 16 ++- test/schema.py | 32 +++--- 12 files changed, 189 insertions(+), 140 deletions(-) delete mode 100644 go/jscfg/jscfg.go diff --git a/go/jscfg/jscfg.go b/go/jscfg/jscfg.go deleted file mode 100644 index 5336f9e188..0000000000 --- a/go/jscfg/jscfg.go +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2012, Google Inc. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package jscfg implements a simple API for reading and writing JSON files. -package jscfg - -import "encoding/json" - -// ToJSON converts a structure to JSON, or panics -func ToJSON(val interface{}) string { - data, err := json.MarshalIndent(val, "", " ") - // This is not strictly the spirit of panic. This is meant to be used - // where it would be a programming error to have json encoding fail. - if err != nil { - panic(err) - } - return string(data) -} diff --git a/go/vt/hook/hook.go b/go/vt/hook/hook.go index cac43d9824..9134349ebb 100644 --- a/go/vt/hook/hook.go +++ b/go/vt/hook/hook.go @@ -14,7 +14,6 @@ import ( "syscall" log "github.com/golang/glog" - "github.com/youtube/vitess/go/jscfg" vtenv "github.com/youtube/vitess/go/vt/env" ) @@ -129,6 +128,30 @@ func (hook *Hook) ExecuteOptional() error { return nil } +// String returns a printable version of the HookResult func (hr *HookResult) String() string { - return jscfg.ToJSON(hr) + result := "result: " + switch hr.ExitStatus { + case HOOK_SUCCESS: + result += "HOOK_SUCCESS" + case HOOK_DOES_NOT_EXIST: + result += "HOOK_DOES_NOT_EXIST" + case HOOK_STAT_FAILED: + result += "HOOK_STAT_FAILED" + case HOOK_CANNOT_GET_EXIT_STATUS: + result += "HOOK_CANNOT_GET_EXIT_STATUS" + case HOOK_INVALID_NAME: + result += "HOOK_INVALID_NAME" + case HOOK_VTROOT_ERROR: + result += "HOOK_VTROOT_ERROR" + default: + result += fmt.Sprintf("exit(%v)", hr.ExitStatus) + } + if hr.Stdout != "" { + result += "\nstdout:\n" + hr.Stdout + } + if hr.Stderr != "" { + result += "\nstderr:\n" + hr.Stderr + } + return result } diff --git a/go/vt/mysqlctl/proto/schema.go b/go/vt/mysqlctl/proto/schema.go index 7f73d3420e..4983091b14 100644 --- a/go/vt/mysqlctl/proto/schema.go +++ b/go/vt/mysqlctl/proto/schema.go @@ -11,7 +11,6 @@ import ( "regexp" "strings" - "github.com/youtube/vitess/go/jscfg" "github.com/youtube/vitess/go/vt/concurrency" ) @@ -59,10 +58,6 @@ type SchemaDefinition struct { Version string } -func (sd *SchemaDefinition) String() string { - return jscfg.ToJSON(sd) -} - // FilterTables returns a copy which includes only // whitelisted tables (tables), no blacklisted tables (excludeTables) and optionally views (includeViews). func (sd *SchemaDefinition) FilterTables(tables, excludeTables []string, includeViews bool) (*SchemaDefinition, error) { @@ -160,13 +155,13 @@ func (sd *SchemaDefinition) GetTable(table string) (td *TableDefinition, ok bool // All SQL statements will have {{.DatabaseName}} in place of the actual db name. func (sd *SchemaDefinition) ToSQLStrings() []string { sqlStrings := make([]string, 0, len(sd.TableDefinitions)+1) - createViewSql := make([]string, 0, len(sd.TableDefinitions)) + createViewSQL := make([]string, 0, len(sd.TableDefinitions)) sqlStrings = append(sqlStrings, sd.DatabaseSchema) for _, td := range sd.TableDefinitions { if td.Type == TableView { - createViewSql = append(createViewSql, td.Schema) + createViewSQL = append(createViewSQL, td.Schema) } else { lines := strings.Split(td.Schema, "\n") for i, line := range lines { @@ -178,7 +173,7 @@ func (sd *SchemaDefinition) ToSQLStrings() []string { } } - return append(sqlStrings, createViewSql...) + return append(sqlStrings, createViewSQL...) } // DiffSchema generates a report on what's different between two SchemaDefinitions @@ -270,7 +265,3 @@ type SchemaChangeResult struct { BeforeSchema *SchemaDefinition AfterSchema *SchemaDefinition } - -func (scr *SchemaChangeResult) String() string { - return jscfg.ToJSON(scr) -} diff --git a/go/vt/mysqlctl/proto/schema_test.go b/go/vt/mysqlctl/proto/schema_test.go index 62f76f1b2f..9a01f5df00 100644 --- a/go/vt/mysqlctl/proto/schema_test.go +++ b/go/vt/mysqlctl/proto/schema_test.go @@ -203,7 +203,7 @@ func TestSchemaDiff(t *testing.T) { testDiff(t, nil, nil, "sd1", "sd2", nil) testDiff(t, sd1, nil, "sd1", "sd2", []string{ - fmt.Sprintf("sd1 and sd2 are different, sd1: %v, sd2: null", sd1), + fmt.Sprintf("sd1 and sd2 are different, sd1: %v, sd2: ", sd1), }) testDiff(t, sd1, sd3, "sd1", "sd3", []string{ diff --git a/go/vt/tabletmanager/actionnode/actionnode.go b/go/vt/tabletmanager/actionnode/actionnode.go index 39bd1710d0..471366bfc6 100644 --- a/go/vt/tabletmanager/actionnode/actionnode.go +++ b/go/vt/tabletmanager/actionnode/actionnode.go @@ -10,12 +10,11 @@ package actionnode import ( + "encoding/json" "fmt" "os" "os/user" "time" - - "github.com/youtube/vitess/go/jscfg" ) const ( @@ -264,19 +263,31 @@ type ActionNode struct { } // ToJSON returns a JSON representation of the object. -func (n *ActionNode) ToJSON() string { - result := jscfg.ToJSON(n) + "\n" +func (n *ActionNode) ToJSON() (string, error) { + data, err := json.MarshalIndent(n, "", " ") + if err != nil { + return "", fmt.Errorf("cannot JSON-marshal node: %v", err) + } + result := string(data) + "\n" if n.Args == nil { result += "{}\n" } else { - result += jscfg.ToJSON(n.Args) + "\n" + data, err := json.MarshalIndent(n.Args, "", " ") + if err != nil { + return "", fmt.Errorf("cannot JSON-marshal node args: %v", err) + } + result += string(data) + "\n" } if n.Reply == nil { result += "{}\n" } else { - result += jscfg.ToJSON(n.Reply) + "\n" + data, err := json.MarshalIndent(n.Reply, "", " ") + if err != nil { + return "", fmt.Errorf("cannot JSON-marshal node reply: %v", err) + } + result += string(data) + "\n" } - return result + return result, nil } // SetGuid will set the ActionGuid field for the action node diff --git a/go/vt/tabletmanager/actionnode/structs_test.go b/go/vt/tabletmanager/actionnode/structs_test.go index 2c2ed076b5..7954bdb46a 100644 --- a/go/vt/tabletmanager/actionnode/structs_test.go +++ b/go/vt/tabletmanager/actionnode/structs_test.go @@ -2,11 +2,9 @@ package actionnode import ( "encoding/json" - "strings" "testing" "github.com/youtube/vitess/go/bson" - "github.com/youtube/vitess/go/jscfg" pb "github.com/youtube/vitess/go/vt/proto/topodata" ) @@ -32,12 +30,13 @@ func TestMissingFieldsJson(t *testing.T) { ExpectedMasterIPAddr: "i1", ScrapStragglers: true, } - data := jscfg.ToJSON(swra) + data, err := json.MarshalIndent(swra, "", " ") + if err != nil { + t.Fatalf("cannot marshal: %v", err) + } output := &SlaveWasRestartedArgs{} - decoder := json.NewDecoder(strings.NewReader(data)) - err := decoder.Decode(output) - if err != nil { + if err = json.Unmarshal(data, output); err != nil { t.Errorf("Cannot re-decode struct without field: %v", err) } } @@ -49,12 +48,13 @@ func TestExtraFieldsJson(t *testing.T) { Cell: "aa", }, } - data := jscfg.ToJSON(swra) + data, err := json.MarshalIndent(swra, "", " ") + if err != nil { + t.Fatalf("cannot marshal: %v", err) + } output := &slaveWasRestartedTestArgs{} - decoder := json.NewDecoder(strings.NewReader(data)) - err := decoder.Decode(output) - if err != nil { + if err = json.Unmarshal(data, output); err != nil { t.Errorf("Cannot re-decode struct without field: %v", err) } } diff --git a/go/vt/tabletmanager/actionnode/utils.go b/go/vt/tabletmanager/actionnode/utils.go index 14ea8b2b37..9b774b4720 100644 --- a/go/vt/tabletmanager/actionnode/utils.go +++ b/go/vt/tabletmanager/actionnode/utils.go @@ -33,7 +33,11 @@ func (n *ActionNode) LockKeyspace(ctx context.Context, ts topo.Server, keyspace span.Annotate("keyspace", keyspace) defer span.Finish() - return ts.LockKeyspaceForAction(ctx, keyspace, n.ToJSON()) + j, err := n.ToJSON() + if err != nil { + return "", err + } + return ts.LockKeyspaceForAction(ctx, keyspace, j) } // UnlockKeyspace unlocks a previously locked keyspace. @@ -60,7 +64,16 @@ func (n *ActionNode) UnlockKeyspace(ctx context.Context, ts topo.Server, keyspac n.Error = "" n.State = ActionStateDone } - err := ts.UnlockKeyspaceForAction(ctx, keyspace, lockPath, n.ToJSON()) + j, err := n.ToJSON() + if err != nil { + if actionError != nil { + // this will be masked + log.Warningf("node.ToJSON failed: %v", err) + return actionError + } + return err + } + err = ts.UnlockKeyspaceForAction(ctx, keyspace, lockPath, j) if actionError != nil { if err != nil { // this will be masked @@ -83,7 +96,11 @@ func (n *ActionNode) LockShard(ctx context.Context, ts topo.Server, keyspace, sh span.Annotate("shard", shard) defer span.Finish() - return ts.LockShardForAction(ctx, keyspace, shard, n.ToJSON()) + j, err := n.ToJSON() + if err != nil { + return "", err + } + return ts.LockShardForAction(ctx, keyspace, shard, j) } // UnlockShard unlocks a previously locked shard. @@ -111,7 +128,16 @@ func (n *ActionNode) UnlockShard(ctx context.Context, ts topo.Server, keyspace, n.Error = "" n.State = ActionStateDone } - err := ts.UnlockShardForAction(ctx, keyspace, shard, lockPath, n.ToJSON()) + j, err := n.ToJSON() + if err != nil { + if actionError != nil { + // this will be masked + log.Warningf("node.ToJSON failed: %v", err) + return actionError + } + return err + } + err = ts.UnlockShardForAction(ctx, keyspace, shard, lockPath, j) if actionError != nil { if err != nil { // this will be masked @@ -135,7 +161,11 @@ func (n *ActionNode) LockSrvShard(ctx context.Context, ts topo.Server, cell, key span.Annotate("cell", cell) defer span.Finish() - return ts.LockSrvShardForAction(ctx, cell, keyspace, shard, n.ToJSON()) + j, err := n.ToJSON() + if err != nil { + return "", err + } + return ts.LockSrvShardForAction(ctx, cell, keyspace, shard, j) } // UnlockSrvShard unlocks a previously locked serving shard. @@ -164,7 +194,16 @@ func (n *ActionNode) UnlockSrvShard(ctx context.Context, ts topo.Server, cell, k n.Error = "" n.State = ActionStateDone } - err := ts.UnlockSrvShardForAction(ctx, cell, keyspace, shard, lockPath, n.ToJSON()) + j, err := n.ToJSON() + if err != nil { + if actionError != nil { + // this will be masked + log.Warningf("node.ToJSON failed: %v", err) + return actionError + } + return err + } + err = ts.UnlockSrvShardForAction(ctx, cell, keyspace, shard, lockPath, j) if actionError != nil { if err != nil { // this will be masked diff --git a/go/vt/vtctl/query.go b/go/vt/vtctl/query.go index 19c0ea3cb7..616e1c86ed 100644 --- a/go/vt/vtctl/query.go +++ b/go/vt/vtctl/query.go @@ -11,7 +11,6 @@ import ( "strings" "time" - "github.com/youtube/vitess/go/jscfg" "github.com/youtube/vitess/go/vt/tabletserver/tabletconn" "github.com/youtube/vitess/go/vt/topo" "github.com/youtube/vitess/go/vt/vtgate/vtgateconn" @@ -123,8 +122,7 @@ func commandVtGateExecute(ctx context.Context, wr *wrangler.Wrangler, subFlags * if err != nil { return fmt.Errorf("Execute failed: %v", err) } - wr.Logger().Printf("%v\n", jscfg.ToJSON(qr)) - return nil + return printJSON(wr, qr) } func commandVtGateExecuteShard(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { @@ -158,8 +156,7 @@ func commandVtGateExecuteShard(ctx context.Context, wr *wrangler.Wrangler, subFl if err != nil { return fmt.Errorf("Execute failed: %v", err) } - wr.Logger().Printf("%v\n", jscfg.ToJSON(qr)) - return nil + return printJSON(wr, qr) } func commandVtGateSplitQuery(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { @@ -185,8 +182,7 @@ func commandVtGateSplitQuery(ctx context.Context, wr *wrangler.Wrangler, subFlag if err != nil { return fmt.Errorf("SplitQuery failed: %v", err) } - wr.Logger().Printf("%v\n", jscfg.ToJSON(r)) - return nil + return printJSON(wr, r) } func commandVtTabletExecute(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { @@ -229,8 +225,7 @@ func commandVtTabletExecute(ctx context.Context, wr *wrangler.Wrangler, subFlags if err != nil { return fmt.Errorf("Execute failed: %v", err) } - wr.Logger().Printf("%v\n", jscfg.ToJSON(qr)) - return nil + return printJSON(wr, qr) } func commandVtTabletStreamHealth(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { diff --git a/go/vt/vtctl/vtctl.go b/go/vt/vtctl/vtctl.go index e01e1412a7..0081e66eba 100644 --- a/go/vt/vtctl/vtctl.go +++ b/go/vt/vtctl/vtctl.go @@ -84,6 +84,7 @@ COMMAND ARGUMENT DEFINITIONS package vtctl import ( + "encoding/json" "errors" "flag" "fmt" @@ -96,7 +97,6 @@ import ( log "github.com/golang/glog" "github.com/youtube/vitess/go/flagutil" - "github.com/youtube/vitess/go/jscfg" "github.com/youtube/vitess/go/netutil" hk "github.com/youtube/vitess/go/vt/hook" "github.com/youtube/vitess/go/vt/logutil" @@ -567,8 +567,8 @@ func parseTabletType(param string, types []pb.TabletType) (pb.TabletType, error) return tabletType, nil } -// parseKeyspaceIdType parses the keyspace id type into the enum -func parseKeyspaceIdType(param string) (pb.KeyspaceIdType, error) { +// parseKeyspaceIDType parses the keyspace id type into the enum +func parseKeyspaceIDType(param string) (pb.KeyspaceIdType, error) { if param == "" { return pb.KeyspaceIdType_UNSET, nil } @@ -660,10 +660,10 @@ func commandGetTablet(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag return err } tabletInfo, err := wr.TopoServer().GetTablet(ctx, tabletAlias) - if err == nil { - wr.Logger().Printf("%v\n", jscfg.ToJSON(tabletInfo)) + if err != nil { + return err } - return err + return printJSON(wr, tabletInfo) } func commandUpdateTabletAddrs(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { @@ -986,10 +986,10 @@ func commandExecuteFetchAsDba(ctx context.Context, wr *wrangler.Wrangler, subFla } query := subFlags.Arg(1) qr, err := wr.ExecuteFetchAsDba(ctx, alias, query, *maxRows, *wantFields, *disableBinlogs, *reloadSchema) - if err == nil { - wr.Logger().Printf("%v\n", jscfg.ToJSON(qr)) + if err != nil { + return err } - return err + return printJSON(wr, qr) } func commandExecuteHook(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { @@ -1006,10 +1006,10 @@ func commandExecuteHook(ctx context.Context, wr *wrangler.Wrangler, subFlags *fl } hook := &hk.Hook{Name: subFlags.Arg(1), Parameters: subFlags.Args()[2:]} hr, err := wr.ExecuteHook(ctx, tabletAlias, hook) - if err == nil { - log.Infof(hr.String()) + if err != nil { + return err } - return err + return printJSON(wr, hr) } func commandCreateShard(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { @@ -1053,10 +1053,10 @@ func commandGetShard(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag. return err } shardInfo, err := wr.TopoServer().GetShard(ctx, keyspace, shard) - if err == nil { - wr.Logger().Printf("%v\n", jscfg.ToJSON(shardInfo)) + if err != nil { + return err } - return err + return printJSON(wr, shardInfo) } func commandRebuildShardGraph(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { @@ -1389,7 +1389,7 @@ func commandCreateKeyspace(ctx context.Context, wr *wrangler.Wrangler, subFlags } keyspace := subFlags.Arg(0) - kit, err := parseKeyspaceIdType(*shardingColumnType) + kit, err := parseKeyspaceIDType(*shardingColumnType) if err != nil { return err } @@ -1453,10 +1453,10 @@ func commandGetKeyspace(ctx context.Context, wr *wrangler.Wrangler, subFlags *fl keyspace := subFlags.Arg(0) keyspaceInfo, err := wr.TopoServer().GetKeyspace(ctx, keyspace) - if err == nil { - wr.Logger().Printf("%v\n", jscfg.ToJSON(keyspaceInfo)) + if err != nil { + return err } - return err + return printJSON(wr, keyspaceInfo) } func commandSetKeyspaceShardingInfo(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { @@ -1477,7 +1477,7 @@ func commandSetKeyspaceShardingInfo(ctx context.Context, wr *wrangler.Wrangler, kit := pb.KeyspaceIdType_UNSET if subFlags.NArg() >= 3 { var err error - kit, err = parseKeyspaceIdType(subFlags.Arg(2)) + kit, err = parseKeyspaceIDType(subFlags.Arg(2)) if err != nil { return err } @@ -1615,11 +1615,10 @@ func commandFindAllShardsInKeyspace(ctx context.Context, wr *wrangler.Wrangler, keyspace := subFlags.Arg(0) result, err := topo.FindAllShardsInKeyspace(ctx, wr.TopoServer(), keyspace) - if err == nil { - wr.Logger().Printf("%v\n", jscfg.ToJSON(result)) + if err != nil { + return err } - return err - + return printJSON(wr, result) } func commandResolve(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { @@ -1741,16 +1740,16 @@ func commandGetSchema(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag } sd, err := wr.GetSchema(ctx, tabletAlias, tableArray, excludeTableArray, *includeViews) - if err == nil { - if *tableNamesOnly { - for _, td := range sd.TableDefinitions { - wr.Logger().Printf("%v\n", td.Name) - } - } else { - wr.Logger().Printf("%v\n", jscfg.ToJSON(sd)) - } + if err != nil { + return err } - return err + if *tableNamesOnly { + for _, td := range sd.TableDefinitions { + wr.Logger().Printf("%v\n", td.Name) + } + return nil + } + return printJSON(wr, sd) } func commandReloadSchema(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { @@ -1824,10 +1823,10 @@ func commandApplySchema(ctx context.Context, wr *wrangler.Wrangler, subFlags *fl return err } scr, err := wr.ApplySchemaKeyspace(ctx, keyspace, change, true, *force, *waitSlaveTimeout) - if err == nil { - log.Infof(scr.String()) + if err != nil { + return err } - return err + return printJSON(wr, scr) } func commandCopySchemaShard(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { @@ -1991,10 +1990,10 @@ func commandGetSrvKeyspace(ctx context.Context, wr *wrangler.Wrangler, subFlags } srvKeyspace, err := wr.TopoServer().GetSrvKeyspace(ctx, subFlags.Arg(0), subFlags.Arg(1)) - if err == nil { - wr.Logger().Printf("%v\n", jscfg.ToJSON(srvKeyspace)) + if err != nil { + return err } - return err + return printJSON(wr, srvKeyspace) } func commandGetSrvKeyspaceNames(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { @@ -2028,10 +2027,10 @@ func commandGetSrvShard(ctx context.Context, wr *wrangler.Wrangler, subFlags *fl return err } srvShard, err := wr.TopoServer().GetSrvShard(ctx, subFlags.Arg(0), keyspace, shard) - if err == nil { - wr.Logger().Printf("%v\n", jscfg.ToJSON(srvShard)) + if err != nil { + return err } - return err + return printJSON(wr, srvShard) } func commandGetEndPoints(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { @@ -2051,10 +2050,10 @@ func commandGetEndPoints(ctx context.Context, wr *wrangler.Wrangler, subFlags *f return err } endPoints, _, err := wr.TopoServer().GetEndPoints(ctx, subFlags.Arg(0), keyspace, shard, tabletType) - if err == nil { - wr.Logger().Printf("%v\n", jscfg.ToJSON(endPoints)) + if err != nil { + return err } - return err + return printJSON(wr, endPoints) } func commandGetShardReplication(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { @@ -2070,10 +2069,10 @@ func commandGetShardReplication(ctx context.Context, wr *wrangler.Wrangler, subF return err } shardReplication, err := wr.TopoServer().GetShardReplication(ctx, subFlags.Arg(0), keyspace, shard) - if err == nil { - wr.Logger().Printf("%v\n", jscfg.ToJSON(shardReplication)) + if err != nil { + return err } - return err + return printJSON(wr, shardReplication) } func commandHelp(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { @@ -2146,6 +2145,17 @@ func sortReplicatingTablets(tablets []*topo.TabletInfo, stats []*myproto.Replica return rtablets } +// printJSON will print the JSON version of the structure to the logger +// console output, or an error to the logger's Error channel. +func printJSON(wr *wrangler.Wrangler, val interface{}) error { + data, err := json.MarshalIndent(val, "", " ") + if err != nil { + return fmt.Errorf("cannot marshal data: %v", err) + } + wr.Logger().Printf("%v\n", string(data)) + return nil +} + // RunCommand will execute the command using the provided wrangler. // It will return the actionPath to wait on for long remote actions if // applicable. diff --git a/go/vt/worker/split_clone.go b/go/vt/worker/split_clone.go index 2dc74157e7..4b284e8b6e 100644 --- a/go/vt/worker/split_clone.go +++ b/go/vt/worker/split_clone.go @@ -14,7 +14,6 @@ import ( "golang.org/x/net/context" "github.com/youtube/vitess/go/event" - "github.com/youtube/vitess/go/jscfg" "github.com/youtube/vitess/go/sync2" "github.com/youtube/vitess/go/vt/binlog/binlogplayer" "github.com/youtube/vitess/go/vt/key" @@ -235,7 +234,7 @@ func (scw *SplitCloneWorker) init(ctx context.Context) error { if os == nil { return fmt.Errorf("the specified shard %v/%v is not in any overlapping shard", scw.keyspace, scw.shard) } - scw.wr.Logger().Infof("Found overlapping shards: %v\n", jscfg.ToJSON(os)) + scw.wr.Logger().Infof("Found overlapping shards: %+v\n", os) // one side should have served types, the other one none, // figure out wich is which, then double check them all diff --git a/go/vt/wrangler/zkns.go b/go/vt/wrangler/zkns.go index 2fa483b8ce..c77586bb12 100644 --- a/go/vt/wrangler/zkns.go +++ b/go/vt/wrangler/zkns.go @@ -1,13 +1,13 @@ package wrangler import ( + "encoding/json" "fmt" "path" "sort" "strings" log "github.com/golang/glog" - "github.com/youtube/vitess/go/jscfg" "github.com/youtube/vitess/go/vt/topo" "github.com/youtube/vitess/go/vt/zktopo" "github.com/youtube/vitess/go/zk" @@ -220,13 +220,19 @@ type LegacyZknsAddrs struct { } func writeAddr(zconn zk.Conn, zkPath string, addr *zkns.ZknsAddr) error { - data := jscfg.ToJSON(addr) - _, err := zk.CreateOrUpdate(zconn, zkPath, data, 0, zookeeper.WorldACL(zookeeper.PERM_ALL), true) + data, err := json.MarshalIndent(addr, "", " ") + if err != nil { + return err + } + _, err = zk.CreateOrUpdate(zconn, zkPath, string(data), 0, zookeeper.WorldACL(zookeeper.PERM_ALL), true) return err } func writeAddrs(zconn zk.Conn, zkPath string, addrs *LegacyZknsAddrs) error { - data := jscfg.ToJSON(addrs) - _, err := zk.CreateOrUpdate(zconn, zkPath, data, 0, zookeeper.WorldACL(zookeeper.PERM_ALL), true) + data, err := json.MarshalIndent(addrs, "", " ") + if err != nil { + return err + } + _, err = zk.CreateOrUpdate(zconn, zkPath, string(data), 0, zookeeper.WorldACL(zookeeper.PERM_ALL), true) return err } diff --git a/test/schema.py b/test/schema.py index 0a8f99f0f9..054938c842 100755 --- a/test/schema.py +++ b/test/schema.py @@ -136,19 +136,13 @@ class TestSchema(unittest.TestCase): tablet.mquery(db_name, 'show tables') def _apply_schema(self, keyspace, sql): - out, _ = utils.run_vtctl(['ApplySchema', - '-sql='+sql, - keyspace], - trap_output=True, - raise_on_error=True) - return out + utils.run_vtctl(['ApplySchema', + '-sql='+sql, + keyspace]) def _get_schema(self, tablet_alias): - out, _ = utils.run_vtctl(['GetSchema', - tablet_alias], - trap_output=True, - raise_on_error=True) - return out + return utils.run_vtctl_json(['GetSchema', + tablet_alias]) def _create_test_table_sql(self, table): return 'CREATE TABLE %s ( \ @@ -222,23 +216,23 @@ class TestSchema(unittest.TestCase): finally: # Include shard2 tablets for tearDown. tablets.extend(tablets_shard2) - + shard_2_master.init_tablet( 'master', 'test_keyspace', '2') shard_2_replica1.init_tablet('replica', 'test_keyspace', '2') - + # We intentionally don't want to create a db on these tablets. shard_2_master.start_vttablet(wait_for_state=None) shard_2_replica1.start_vttablet(wait_for_state=None) - + shard_2_master.wait_for_vttablet_state('NOT_SERVING') shard_2_replica1.wait_for_vttablet_state('NOT_SERVING') - + for t in tablets_shard2: t.reset_replication() utils.run_vtctl(['InitShardMaster', test_keyspace+'/2', shard_2_master.tablet_alias], auto_log=True) utils.run_vtctl(['ValidateKeyspace', '-ping-tablets', test_keyspace]) - + def test_vtctl_copyschemashard_use_tablet_as_source(self): self._test_vtctl_copyschemashard(shard_0_master.tablet_alias) @@ -247,9 +241,9 @@ class TestSchema(unittest.TestCase): def _test_vtctl_copyschemashard(self, source): self._apply_initial_schema() - + self._setUp_tablets_shard_2() - + # CopySchemaShard is responsible for creating the db; one shouldn't exist before # the command is run. self._check_db_not_created(shard_2_master) @@ -261,7 +255,7 @@ class TestSchema(unittest.TestCase): source, 'test_keyspace/2'], auto_log=True) - + # shard_2_master should look the same as the replica we copied from self._check_tables(shard_2_master, 4) utils.wait_for_replication_pos(shard_2_master, shard_2_replica1) From ca77d9a5d21a50339bdce50f01f54f91c8d99c28 Mon Sep 17 00:00:00 2001 From: Alain Jobart Date: Mon, 17 Aug 2015 08:34:18 -0700 Subject: [PATCH 13/80] Retiring zkclient2. toposerver API is going away. --- Makefile | 6 +- go/cmd/zkclient2/plugin_influxdbbackend.go | 11 -- go/cmd/zkclient2/zkclient2.go | 198 --------------------- test/client_test.py | 1 - test/config.json | 3 - test/vtgatev2_test.py | 1 - test/zkocc_test.py | 163 ----------------- 7 files changed, 2 insertions(+), 381 deletions(-) delete mode 100644 go/cmd/zkclient2/plugin_influxdbbackend.go delete mode 100644 go/cmd/zkclient2/zkclient2.go delete mode 100755 test/zkocc_test.py diff --git a/Makefile b/Makefile index 4d9a5c3af8..d62da6e61f 100644 --- a/Makefile +++ b/Makefile @@ -86,8 +86,7 @@ site_integration_test_files = \ tabletmanager.py \ update_stream.py \ vtdb_test.py \ - vtgatev2_test.py \ - zkocc_test.py + vtgatev2_test.py # These tests should be run by developers after making code changes. # Integration tests are grouped into 3 suites. @@ -111,8 +110,7 @@ small_integration_test_files = \ update_stream.py \ custom_sharding.py \ initial_sharding_bytes.py \ - initial_sharding.py \ - zkocc_test.py + initial_sharding.py # TODO(mberlin): Remove -v option to worker.py when we found out what causes 10 minute Travis timeouts. medium_integration_test_files = \ diff --git a/go/cmd/zkclient2/plugin_influxdbbackend.go b/go/cmd/zkclient2/plugin_influxdbbackend.go deleted file mode 100644 index 0c1605e4ea..0000000000 --- a/go/cmd/zkclient2/plugin_influxdbbackend.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright 2014, Google Inc. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -// This plugin imports influxdbbackend to register the influxdbbackend stats backend. - -import ( - _ "github.com/youtube/vitess/go/stats/influxdbbackend" -) diff --git a/go/cmd/zkclient2/zkclient2.go b/go/cmd/zkclient2/zkclient2.go deleted file mode 100644 index ba03f1a9bc..0000000000 --- a/go/cmd/zkclient2/zkclient2.go +++ /dev/null @@ -1,198 +0,0 @@ -// Copyright 2012, Google Inc. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "flag" - "fmt" - "os" - "runtime/pprof" - "sort" - "time" - - "golang.org/x/net/context" - - log "github.com/golang/glog" - "github.com/youtube/vitess/go/exit" - "github.com/youtube/vitess/go/rpcplus" - "github.com/youtube/vitess/go/rpcwrap/bsonrpc" - "github.com/youtube/vitess/go/sync2" - "github.com/youtube/vitess/go/vt/logutil" - "github.com/youtube/vitess/go/vt/topo" - - pb "github.com/youtube/vitess/go/vt/proto/topodata" -) - -var ( - usage = ` -Queries the topo server, for test purposes. -` - mode = flag.String("mode", "get", "which operation to run on the node (getSrvKeyspaceNames, getSrvKeyspace, getEndPoints, qps)") - server = flag.String("server", "localhost:3801", "topo server to dial") - timeout = flag.Duration("timeout", 5*time.Second, "connection timeout") - // flag can't overlap with servenv's cpu_profile - cpuProfile = flag.String("zkclient_cpu_profile", "", "write cpu profile to file") -) - -func init() { - flag.Usage = func() { - fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0]) - flag.PrintDefaults() - fmt.Fprintf(os.Stderr, usage) - } -} - -func connect() *rpcplus.Client { - rpcClient, err := bsonrpc.DialHTTP("tcp", *server, *timeout) - if err != nil { - log.Fatalf("Can't connect to topo server: %v", err) - } - return rpcClient -} - -func getSrvKeyspaceNames(ctx context.Context, rpcClient *rpcplus.Client, cell string, verbose bool) { - req := &topo.GetSrvKeyspaceNamesArgs{ - Cell: cell, - } - reply := &topo.SrvKeyspaceNames{} - if err := rpcClient.Call(ctx, "TopoReader.GetSrvKeyspaceNames", req, reply); err != nil { - log.Fatalf("TopoReader.GetSrvKeyspaceNames error: %v", err) - } - if verbose { - for i, entry := range reply.Entries { - println(fmt.Sprintf("KeyspaceNames[%v] = %v", i, entry)) - } - } -} - -func getSrvKeyspace(ctx context.Context, rpcClient *rpcplus.Client, cell, keyspace string, verbose bool) { - req := &topo.GetSrvKeyspaceArgs{ - Cell: cell, - Keyspace: keyspace, - } - reply := &topo.SrvKeyspace{} - if err := rpcClient.Call(ctx, "TopoReader.GetSrvKeyspace", req, reply); err != nil { - log.Fatalf("TopoReader.GetSrvKeyspace error: %v", err) - } - if verbose { - tabletTypes := make([]string, 0, len(reply.Partitions)) - for t := range reply.Partitions { - tabletTypes = append(tabletTypes, string(t)) - } - sort.Strings(tabletTypes) - for _, t := range tabletTypes { - println(fmt.Sprintf("Partitions[%v] =", t)) - for i, s := range reply.Partitions[topo.TabletType(t)].ShardReferences { - println(fmt.Sprintf(" ShardReferences[%v]=%v", i, s.KeyRange.String())) - } - } - } -} - -func getEndPoints(ctx context.Context, rpcClient *rpcplus.Client, cell, keyspace, shard, tabletType string, verbose bool) { - req := &topo.GetEndPointsArgs{ - Cell: cell, - Keyspace: keyspace, - Shard: shard, - TabletType: topo.TabletType(tabletType), - } - reply := &pb.EndPoints{} - if err := rpcClient.Call(ctx, "TopoReader.GetEndPoints", req, reply); err != nil { - log.Fatalf("TopoReader.GetEndPoints error: %v", err) - } - if verbose { - for i, e := range reply.Entries { - println(fmt.Sprintf("Entries[%v] = %v %v", i, e.Uid, e.Host)) - } - } -} - -// qps is a function used by tests to run a vtgate load check. -// It will get the same srvKeyspaces as fast as possible and display the QPS. -func qps(ctx context.Context, cell string, keyspaces []string) { - var count sync2.AtomicInt32 - for _, keyspace := range keyspaces { - for i := 0; i < 10; i++ { - go func() { - rpcClient := connect() - for true { - getSrvKeyspace(ctx, rpcClient, cell, keyspace, false) - count.Add(1) - } - }() - } - } - - ticker := time.NewTicker(time.Second) - i := 0 - for _ = range ticker.C { - c := count.Get() - count.Set(0) - println(fmt.Sprintf("QPS = %v", c)) - i++ - if i == 10 { - break - } - } -} - -func main() { - defer exit.Recover() - defer logutil.Flush() - - flag.Parse() - args := flag.Args() - if len(args) == 0 { - flag.Usage() - exit.Return(1) - } - - if *cpuProfile != "" { - f, err := os.Create(*cpuProfile) - if err != nil { - log.Error(err) - exit.Return(1) - } - pprof.StartCPUProfile(f) - defer pprof.StopCPUProfile() - } - - ctx := context.Background() - if *mode == "getSrvKeyspaceNames" { - rpcClient := connect() - if len(args) == 1 { - getSrvKeyspaceNames(ctx, rpcClient, args[0], true) - } else { - log.Errorf("getSrvKeyspaceNames only takes one argument") - exit.Return(1) - } - - } else if *mode == "getSrvKeyspace" { - rpcClient := connect() - if len(args) == 2 { - getSrvKeyspace(ctx, rpcClient, args[0], args[1], true) - } else { - log.Errorf("getSrvKeyspace only takes two arguments") - exit.Return(1) - } - - } else if *mode == "getEndPoints" { - rpcClient := connect() - if len(args) == 4 { - getEndPoints(ctx, rpcClient, args[0], args[1], args[2], args[3], true) - } else { - log.Errorf("getEndPoints only takes four arguments") - exit.Return(1) - } - - } else if *mode == "qps" { - qps(ctx, args[0], args[1:]) - - } else { - flag.Usage() - log.Errorf("Invalid mode: %v", *mode) - exit.Return(1) - } -} diff --git a/test/client_test.py b/test/client_test.py index f3ac607b1e..c26e6121ef 100755 --- a/test/client_test.py +++ b/test/client_test.py @@ -32,7 +32,6 @@ from vtdb import shard_constants from vtdb import vtdb_logger from vtdb import vtgatev2 from vtdb import vtgate_cursor -from zk import zkocc conn_class = vtgatev2 __tablets = None diff --git a/test/config.json b/test/config.json index a159220edf..7cf5a9ac49 100644 --- a/test/config.json +++ b/test/config.json @@ -71,9 +71,6 @@ "vtgatev2": { "File": "vtgatev2_test.py" }, - "zkocc": { - "File": "zkocc_test.py" - }, "initial_sharding_bytes": { "File": "initial_sharding_bytes.py" }, diff --git a/test/vtgatev2_test.py b/test/vtgatev2_test.py index 8377725e04..3ca6537fb4 100755 --- a/test/vtgatev2_test.py +++ b/test/vtgatev2_test.py @@ -28,7 +28,6 @@ from vtdb import dbexceptions from vtdb import vtdb_logger from vtdb import vtgatev2 from vtdb import vtgate_cursor -from zk import zkocc conn_class = vtgatev2 diff --git a/test/zkocc_test.py b/test/zkocc_test.py deleted file mode 100755 index e13d511714..0000000000 --- a/test/zkocc_test.py +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env python -import datetime -import json -import logging -import os -import re -import tempfile -import time -import unittest - -import environment -import tablet -import utils -from zk import zkocc - - -# We check in this test that we can achieve at least this QPS. -# Sometimes on slow machines this won't work. -# We used to observe 30k+ QPS on workstations. This has gone down to 13k. -# So now the value we check is 5k, and we have an action item to look into it. -MIN_QPS = 5000 - - -def setUpModule(): - try: - environment.topo_server().setup() - - except: - tearDownModule() - raise - - -def tearDownModule(): - if utils.options.skip_teardown: - return - - environment.topo_server().teardown() - utils.kill_sub_processes() - utils.remove_tmp_files() - - -class TopoOccTest(unittest.TestCase): - def setUp(self): - environment.topo_server().wipe() - utils.VtGate().start() - - def tearDown(self): - utils.vtgate.kill() - - def rebuild(self): - utils.run_vtctl(['RebuildKeyspaceGraph', '-rebuild_srv_shards', 'test_keyspace'], auto_log=True) - - def test_get_srv_keyspace_names(self): - utils.run_vtctl('CreateKeyspace test_keyspace1') - utils.run_vtctl('CreateKeyspace test_keyspace2') - t1 = tablet.Tablet(tablet_uid=1, cell="nj") - t1.init_tablet("master", "test_keyspace1", "0") - t1.update_addrs() - t2 = tablet.Tablet(tablet_uid=2, cell="nj") - t2.init_tablet("master", "test_keyspace2", "0") - t2.update_addrs() - utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace*'], auto_log=True) - - # vtgate API test - out, err = utils.run(environment.binary_argstr('zkclient2')+' -server localhost:%d -mode getSrvKeyspaceNames test_nj' % utils.vtgate.port, trap_output=True) - self.assertEqual(err, "KeyspaceNames[0] = test_keyspace1\n" + - "KeyspaceNames[1] = test_keyspace2\n") - - def test_get_srv_keyspace(self): - utils.run_vtctl('CreateKeyspace test_keyspace') - t = tablet.Tablet(tablet_uid=1, cell="nj") - t.init_tablet("master", "test_keyspace", "0") - utils.run_vtctl('UpdateTabletAddrs -hostname localhost -ip-addr 127.0.0.1 -mysql-port %s %s' % (t.mysql_port, t.tablet_alias)) - self.rebuild() - - # vtgate zk API test - out, err = utils.run(environment.binary_argstr('zkclient2')+' -server localhost:%d -mode getSrvKeyspace test_nj test_keyspace' % utils.vtgate.port, trap_output=True) - self.assertEqual(err, "Partitions[master] =\n" + - " ShardReferences[0]={Start: , End: }\n" + - "Partitions[rdonly] =\n" + - " ShardReferences[0]={Start: , End: }\n" + - "Partitions[replica] =\n" + - " ShardReferences[0]={Start: , End: }\n", - "Got wrong content: %s" % err) - - def test_get_end_points(self): - utils.run_vtctl('CreateKeyspace test_keyspace') - t = tablet.Tablet(tablet_uid=1, cell="nj") - t.init_tablet("master", "test_keyspace", "0") - t.update_addrs() - self.rebuild() - - # vtgate zk API test - out, err = utils.run(environment.binary_argstr('zkclient2')+' -server localhost:%d -mode getEndPoints test_nj test_keyspace 0 master' % utils.vtgate.port, trap_output=True) - self.assertEqual(err, "Entries[0] = 1 localhost\n") - - -class TestTopo(unittest.TestCase): - longMessage = True - def setUp(self): - environment.topo_server().wipe() - - # test_vtgate_qps can be run to profile vtgate: - # Just run: - # ./zkocc_test.py -v TestTopo.test_vtgate_qps --skip-teardown - # Then run: - # go tool pprof $VTROOT/bin/vtgate $VTDATAROOT/tmp/vtgate.pprof - # (or with zkclient2 for the client side) - # and for instance type 'web' in the prompt. - def test_vtgate_qps(self): - # create the topology - utils.run_vtctl('CreateKeyspace test_keyspace') - t = tablet.Tablet(tablet_uid=1, cell="nj") - t.init_tablet("master", "test_keyspace", "0") - t.update_addrs() - utils.run_vtctl('RebuildKeyspaceGraph test_keyspace', auto_log=True) - - # start vtgate and the qps-er - utils.VtGate().start( - extra_args=['-cpu_profile', os.path.join(environment.tmproot, - 'vtgate.pprof')]) - qpser = utils.run_bg(environment.binary_args('zkclient2') + [ - '-server', utils.vtgate.addr(), - '-mode', 'qps', - '-zkclient_cpu_profile', os.path.join(environment.tmproot, 'zkclient2.pprof'), - 'test_nj', 'test_keyspace']) - qpser.wait() - - # get the vtgate vars, make sure we have what we need - v = utils.vtgate.get_vars() - - # some checks on performance / stats - rpcCalls = v['TopoReaderRpcQueryCount']['test_nj'] - if rpcCalls < MIN_QPS * 10: - self.fail('QPS is too low: %d < %d' % (rpcCalls / 10, MIN_QPS)) - else: - logging.debug("Recorded qps: %d", rpcCalls / 10) - utils.vtgate.kill() - - def test_fake_zkocc_connection(self): - fkc = zkocc.FakeZkOccConnection.from_data_path("testing", environment.vttop + "/test/fake_zkocc_config.json") - fkc.replace_zk_data("3306", "3310") - fkc.replace_zk_data("127.0.0.1", "my.cool.hostname") - - # new style API tests - keyspaces = fkc.get_srv_keyspace_names('testing') - self.assertEqual(keyspaces, ["test_keyspace"], "get_srv_keyspace_names doesn't work") - keyspace = fkc.get_srv_keyspace('testing', 'test_keyspace') - self.assertEqual({ - 'ShardReferences': [{ - 'KeyRange': {'End': '\xd0', 'Start': '\xc0'}, - 'Name': 'c0-d0'}]}, - keyspace, "keyspace reading is wrong") - end_points = fkc.get_end_points("testing", "test_keyspace", "0", "master") - self.assertEqual({ - 'entries': [{'host': 'my.cool.hostname', - 'named_port_map': {'mysql': 3310, 'vt': 6711}, - 'port': 0, - 'uid': 0}]}, - end_points, "end points are wrong") - -if __name__ == '__main__': - utils.main() From 78d5dc67f0cf889123555416bb2222915504f79f Mon Sep 17 00:00:00 2001 From: Alain Jobart Date: Mon, 17 Aug 2015 08:42:56 -0700 Subject: [PATCH 14/80] This test is not using zkocc any more. --- test/keyspace_test.py | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/test/keyspace_test.py b/test/keyspace_test.py index 490247e1b5..db5782724b 100755 --- a/test/keyspace_test.py +++ b/test/keyspace_test.py @@ -4,20 +4,14 @@ # Use of this source code is governed by a BSD-style license that can # be found in the LICENSE file. -import logging -import threading -import struct -import time import unittest -from vtdb import keyrange_constants -from vtdb import keyspace +from vtdb import vtgate_client import environment import utils import tablet - -from zk import zkocc +from protocols_flavor import protocols_flavor SHARDED_KEYSPACE = "TEST_KEYSPACE_SHARDED" UNSHARDED_KEYSPACE = "TEST_KEYSPACE_UNSHARDED" @@ -172,10 +166,13 @@ def setup_unsharded_keyspace(): ALL_DB_TYPES = ['master', 'rdonly', 'replica'] class TestKeyspace(unittest.TestCase): - def _read_keyspace(self, keyspace_name): - vtgate_client = zkocc.ZkOccConnection(utils.vtgate.addr(), - "test_nj", 30.0) - return keyspace.read_keyspace(vtgate_client, keyspace_name) + def _read_srv_keyspace(self, keyspace_name): + addr = utils.vtgate.rpc_endpoint() + protocol = protocols_flavor().vtgate_python_protocol() + conn = vtgate_client.connect(protocol, addr, 30.0) + result = conn.get_srv_keyspace(keyspace_name) + conn.close() + return result def test_get_keyspace(self): ki = utils.run_vtctl_json(['GetKeyspace', UNSHARDED_KEYSPACE]) @@ -283,27 +280,27 @@ class TestKeyspace(unittest.TestCase): utils.run_vtctl(['DeleteKeyspace', '-recursive', 'test_delete_keyspace']) def test_shard_count(self): - sharded_ks = self._read_keyspace(SHARDED_KEYSPACE) + sharded_ks = self._read_srv_keyspace(SHARDED_KEYSPACE) for db_type in ALL_DB_TYPES: self.assertEqual(sharded_ks.get_shard_count(db_type), 2) - unsharded_ks = self._read_keyspace(UNSHARDED_KEYSPACE) + unsharded_ks = self._read_srv_keyspace(UNSHARDED_KEYSPACE) for db_type in ALL_DB_TYPES: self.assertEqual(unsharded_ks.get_shard_count(db_type), 1) def test_shard_names(self): - sharded_ks = self._read_keyspace(SHARDED_KEYSPACE) + sharded_ks = self._read_srv_keyspace(SHARDED_KEYSPACE) for db_type in ALL_DB_TYPES: self.assertEqual(sharded_ks.get_shard_names(db_type), ['-80', '80-']) - unsharded_ks = self._read_keyspace(UNSHARDED_KEYSPACE) + unsharded_ks = self._read_srv_keyspace(UNSHARDED_KEYSPACE) for db_type in ALL_DB_TYPES: self.assertEqual(unsharded_ks.get_shard_names(db_type), ['0']) def test_keyspace_id_to_shard_name(self): - sharded_ks = self._read_keyspace(SHARDED_KEYSPACE) + sharded_ks = self._read_srv_keyspace(SHARDED_KEYSPACE) for _, sn in enumerate(shard_names): for keyspace_id in shard_kid_map[sn]: self.assertEqual(sharded_ks.keyspace_id_to_shard_name_for_db_type(keyspace_id, 'master'), sn) - unsharded_ks = self._read_keyspace(UNSHARDED_KEYSPACE) + unsharded_ks = self._read_srv_keyspace(UNSHARDED_KEYSPACE) for keyspace_id in shard_kid_map[sn]: self.assertEqual(unsharded_ks.keyspace_id_to_shard_name_for_db_type(keyspace_id, 'master'), '0') From d172b1e637699cee108fcfefdd5064839d190db2 Mon Sep 17 00:00:00 2001 From: Alain Jobart Date: Mon, 17 Aug 2015 09:28:53 -0700 Subject: [PATCH 15/80] Refactoring rowcache_invalidator.py to not use python vttablet client. --- test/rowcache_invalidator.py | 48 ++++++++++-------------------------- test/tablet.py | 4 +-- test/utils.py | 14 +++++++---- 3 files changed, 24 insertions(+), 42 deletions(-) diff --git a/test/rowcache_invalidator.py b/test/rowcache_invalidator.py index 7c14e6d162..32fe814e6c 100755 --- a/test/rowcache_invalidator.py +++ b/test/rowcache_invalidator.py @@ -11,10 +11,6 @@ import time import unittest import urllib2 -from zk import zkocc -from vtdb import topology -from vtdb import vtclient - import environment import framework import tablet @@ -39,6 +35,9 @@ def setUpModule(): replica_tablet.init_mysql()] utils.wait_procs(setup_procs) + # start a vtctld so the vtctl insert commands are just RPCs, not forks + utils.Vtctld().start() + # Start up a master mysql and vttablet logging.debug('Setting up tablets') utils.run_vtctl(['CreateKeyspace', 'test_keyspace']) @@ -85,9 +84,6 @@ def tearDownModule(): class RowCacheInvalidator(unittest.TestCase): def setUp(self): - self.vtgate_client = zkocc.ZkOccConnection(utils.vtgate.addr(), - 'test_nj', 30.0) - topology.read_topology(self.vtgate_client) self.perform_insert(400) def tearDown(self): @@ -103,10 +99,10 @@ class RowCacheInvalidator(unittest.TestCase): def perform_insert(self, count): for i in xrange(count): - self._exec_vt_txn(["insert into vt_insert_test (msg) values ('test %s')" % i]) + self._exec_vt_txn("insert into vt_insert_test (msg) values ('test %s')" % i) def perform_delete(self): - self._exec_vt_txn(['delete from vt_insert_test',]) + self._exec_vt_txn('delete from vt_insert_test') def _wait_for_replica(self): master_position = utils.mysql_query(master_tablet.tablet_uid, @@ -148,20 +144,20 @@ class RowCacheInvalidator(unittest.TestCase): def test_outofband_statements(self): start = self.replica_vars()['InternalErrors'].get('Invalidation', 0) - self._exec_vt_txn(["insert into vt_insert_test (id, msg) values (1000000, 'start')"]) + self._exec_vt_txn("insert into vt_insert_test (id, msg) values (1000000, 'start')") self._wait_for_replica() time.sleep(1.0) # Test update statement result = self._exec_replica_query('select * from vt_insert_test where id = 1000000') - self.assertEqual(result, [(1000000, 'start')]) + self.assertEqual(result, [['1000000', 'start']]) utils.mysql_write_query(master_tablet.tablet_uid, 'vt_test_keyspace', "update vt_insert_test set msg = 'foo' where id = 1000000") self._wait_for_replica() time.sleep(1.0) result = self._exec_replica_query('select * from vt_insert_test where id = 1000000') - self.assertEqual(result, [(1000000, 'foo')]) + self.assertEqual(result, [['1000000', 'foo']]) end1 = self.replica_vars()['InternalErrors'].get('Invalidation', 0) self.assertEqual(start, end1) @@ -183,7 +179,7 @@ class RowCacheInvalidator(unittest.TestCase): self._wait_for_replica() time.sleep(1.0) result = self._exec_replica_query('select * from vt_insert_test where id = 1000000') - self.assertEqual(result, [(1000000, 'bar')]) + self.assertEqual(result, [['1000000', 'bar']]) end3 = self.replica_vars()['InternalErrors'].get('Invalidation', 0) self.assertEqual(end2, end3) @@ -292,30 +288,12 @@ class RowCacheInvalidator(unittest.TestCase): # and restore the type utils.run_vtctl(['ChangeSlaveType', replica_tablet.tablet_alias, 'replica']) - def _vtdb_conn(self): - conn = vtclient.VtOCCConnection(self.vtgate_client, 'test_keyspace', '0', - 'master', 30) - conn.connect() - return conn - - def _exec_vt_txn(self, query_list=None): - if query_list is None: - return - vtdb_conn = self._vtdb_conn() - vtdb_cursor = vtdb_conn.cursor() - vtdb_conn.begin() - for q in query_list: - vtdb_cursor.execute(q, {}) - vtdb_conn.commit() + def _exec_vt_txn(self, query): + master_tablet.execute(query, auto_log=False) def _exec_replica_query(self, query): - conn = vtclient.VtOCCConnection(self.vtgate_client, 'test_keyspace', '0', - 'replica', 30) - conn.connect() - cursor = conn.cursor() - cursor.execute(query, {}) - conn.close() - return list(cursor) + result = replica_tablet.execute(query, auto_log=False) + return result['Rows'] if __name__ == '__main__': diff --git a/test/tablet.py b/test/tablet.py index 258e418869..f637bf1bd7 100644 --- a/test/tablet.py +++ b/test/tablet.py @@ -698,7 +698,7 @@ class Tablet(object): if Tablet.tablets_running > 0: raise utils.TestError('This test is not killing all its vttablets') - def execute(self, sql, bindvars=None, transaction_id=None): + def execute(self, sql, bindvars=None, transaction_id=None, auto_log=True): """execute uses 'vtctl VtTabletExecute' to execute a command. """ args = [ @@ -711,7 +711,7 @@ class Tablet(object): if transaction_id: args.extend(['-transaction_id', str(transaction_id)]) args.extend([self.tablet_alias, sql]) - return utils.run_vtctl_json(args) + return utils.run_vtctl_json(args, auto_log=auto_log) def kill_tablets(tablets): diff --git a/test/utils.py b/test/utils.py index 92825fbc79..6cc24d25d2 100644 --- a/test/utils.py +++ b/test/utils.py @@ -596,8 +596,11 @@ def run_vtctl(clargs, auto_log=False, expect_fail=False, result = vtctld.vtctl_client(clargs) return result, "" elif mode == VTCTL_RPC: - logging.debug("vtctl: %s", " ".join(clargs)) - result = vtctl_client.execute_vtctl_command(vtctld_connection, clargs, info_to_debug=True, action_timeout=120) + if auto_log: + logging.debug("vtctl: %s", " ".join(clargs)) + result = vtctl_client.execute_vtctl_command(vtctld_connection, clargs, + info_to_debug=True, + action_timeout=120) return result, "" raise Exception('Unknown mode: %s', mode) @@ -625,8 +628,8 @@ def run_vtctl_vtctl(clargs, auto_log=False, expect_fail=False, # run_vtctl_json runs the provided vtctl command and returns the result # parsed as json -def run_vtctl_json(clargs): - stdout, stderr = run_vtctl(clargs, trap_output=True, auto_log=True) +def run_vtctl_json(clargs, auto_log=True): + stdout, stderr = run_vtctl(clargs, trap_output=True, auto_log=auto_log) return json.loads(stdout) def get_log_level(): @@ -920,8 +923,9 @@ class Vtctld(object): args.extend(['-service_map', ",".join(protocols_flavor().service_map())]) if protocols_flavor().vtctl_client_protocol() == 'grpc': args.extend(['-grpc_port', str(self.grpc_port)]) + stdout_fd = open(os.path.join(environment.tmproot, 'vtctld.stdout'), 'w') stderr_fd = open(os.path.join(environment.tmproot, 'vtctld.stderr'), 'w') - self.proc = run_bg(args, stderr=stderr_fd) + self.proc = run_bg(args, stdout=stdout_fd, stderr=stderr_fd) # wait for the process to listen to RPC timeout = 30 From 09bff85552e5c4220b8c38733bb1f54bacee8660 Mon Sep 17 00:00:00 2001 From: Shengzhe Yao Date: Tue, 11 Aug 2015 21:12:16 -0700 Subject: [PATCH 16/80] make SplitQuery support VARBINARY column. 1. SplitQuery will split a VARBINARY column as a hex string. That is, it assumes the key range is [0x0000000, 0xFFFFFFFF] and then divides the range into intervals based on split count. 2. Introduce splitBoundariesStringColumn to handle string column case. 3. Refactor existing SplitQuery in sqlquery.go: create two helper functions getColumnType and getColumnMinMax. --- go/sqltypes/sqltypes.go | 12 ++ go/vt/tabletserver/query_splitter.go | 115 +++++++---- go/vt/tabletserver/query_splitter_test.go | 220 +++++++++++++++++++--- go/vt/tabletserver/sqlquery.go | 46 ++++- go/vt/tabletserver/sqlquery_test.go | 48 ++++- 5 files changed, 367 insertions(+), 74 deletions(-) diff --git a/go/sqltypes/sqltypes.go b/go/sqltypes/sqltypes.go index 2aa11b88fd..0012aeac60 100644 --- a/go/sqltypes/sqltypes.go +++ b/go/sqltypes/sqltypes.go @@ -104,6 +104,18 @@ func (v Value) ParseUint64() (val uint64, err error) { return strconv.ParseUint(string(n.raw()), 10, 64) } +// ParseFloat64 will parse a Fractional value into an float64 +func (v Value) ParseFloat64() (val float64, err error) { + if v.Inner == nil { + return 0, fmt.Errorf("value is null") + } + n, ok := v.Inner.(Fractional) + if !ok { + return 0, fmt.Errorf("value is not Fractional") + } + return strconv.ParseFloat(string(n.raw()), 64) +} + // EncodeSql encodes the value into an SQL statement. Can be binary. func (v Value) EncodeSql(b BinWriter) { if v.Inner == nil { diff --git a/go/vt/tabletserver/query_splitter.go b/go/vt/tabletserver/query_splitter.go index 40af9db2e4..e734d11821 100644 --- a/go/vt/tabletserver/query_splitter.go +++ b/go/vt/tabletserver/query_splitter.go @@ -1,6 +1,7 @@ package tabletserver import ( + "encoding/binary" "fmt" "strconv" @@ -26,6 +27,11 @@ type QuerySplitter struct { rowCount int64 } +const ( + startBindVarName = ":_splitquery_start" + endBindVarName = ":_splitquery_end" +) + // NewQuerySplitter creates a new QuerySplitter. query is the original query // to split and splitCount is the desired number of splits. splitCount must // be a positive int, if not it will be set to 1. @@ -95,8 +101,8 @@ func (qs *QuerySplitter) validateQuery() error { // split splits the query into multiple queries. validateQuery() must return // nil error before split() is called. -func (qs *QuerySplitter) split(pkMinMax *mproto.QueryResult) ([]proto.QuerySplit, error) { - boundaries, err := qs.splitBoundaries(pkMinMax) +func (qs *QuerySplitter) split(columnType int64, pkMinMax *mproto.QueryResult) ([]proto.QuerySplit, error) { + boundaries, err := qs.splitBoundaries(columnType, pkMinMax) if err != nil { return nil, err } @@ -108,51 +114,59 @@ func (qs *QuerySplitter) split(pkMinMax *mproto.QueryResult) ([]proto.QuerySplit } splits = append(splits, *split) } else { + boundaries = append(boundaries, sqltypes.Value{}) + whereClause := qs.sel.Where // Loop through the boundaries and generated modified where clauses start := sqltypes.Value{} - clauses := []*sqlparser.Where{} for _, end := range boundaries { - clauses = append(clauses, qs.getWhereClause(start, end)) - start.Inner = end.Inner - } - clauses = append(clauses, qs.getWhereClause(start, sqltypes.Value{})) - // Generate one split per clause - for _, clause := range clauses { - sel := qs.sel - sel.Where = clause + bindVars := make(map[string]interface{}, len(qs.query.BindVariables)) + for k, v := range qs.query.BindVariables { + bindVars[k] = v + } + qs.sel.Where = qs.getWhereClause(whereClause, bindVars, start, end) q := &proto.BoundQuery{ - Sql: sqlparser.String(sel), - BindVariables: qs.query.BindVariables, + Sql: sqlparser.String(qs.sel), + BindVariables: bindVars, } split := &proto.QuerySplit{ Query: *q, RowCount: qs.rowCount, } splits = append(splits, *split) + start.Inner = end.Inner } + qs.sel.Where = whereClause // reset where clause } return splits, err } // getWhereClause returns a whereClause based on desired upper and lower // bounds for primary key. -func (qs *QuerySplitter) getWhereClause(start, end sqltypes.Value) *sqlparser.Where { +func (qs *QuerySplitter) getWhereClause(whereClause *sqlparser.Where, bindVars map[string]interface{}, start, end sqltypes.Value) *sqlparser.Where { var startClause *sqlparser.ComparisonExpr var endClause *sqlparser.ComparisonExpr var clauses sqlparser.BoolExpr // No upper or lower bound, just return the where clause of original query if start.IsNull() && end.IsNull() { - return qs.sel.Where + return whereClause } pk := &sqlparser.ColName{ Name: sqlparser.SQLName(qs.splitColumn), } - // splitColumn >= start if !start.IsNull() { startClause = &sqlparser.ComparisonExpr{ Operator: sqlparser.AST_GE, Left: pk, - Right: sqlparser.NumVal((start).Raw()), + Right: sqlparser.ValArg([]byte(startBindVarName)), + } + if start.IsNumeric() { + v, _ := start.ParseInt64() + bindVars[startBindVarName] = v + } else if start.IsString() { + bindVars[startBindVarName] = start.Raw() + } else if start.IsFractional() { + v, _ := start.ParseFloat64() + bindVars[startBindVarName] = v } } // splitColumn < end @@ -160,7 +174,16 @@ func (qs *QuerySplitter) getWhereClause(start, end sqltypes.Value) *sqlparser.Wh endClause = &sqlparser.ComparisonExpr{ Operator: sqlparser.AST_LT, Left: pk, - Right: sqlparser.NumVal((end).Raw()), + Right: sqlparser.ValArg([]byte(endBindVarName)), + } + if end.IsNumeric() { + v, _ := end.ParseInt64() + bindVars[endBindVarName] = v + } else if end.IsString() { + bindVars[endBindVarName] = end.Raw() + } else if end.IsFractional() { + v, _ := end.ParseFloat64() + bindVars[endBindVarName] = v } } if startClause == nil { @@ -176,9 +199,9 @@ func (qs *QuerySplitter) getWhereClause(start, end sqltypes.Value) *sqlparser.Wh } } } - if qs.sel.Where != nil { + if whereClause != nil { clauses = &sqlparser.AndExpr{ - Left: qs.sel.Where.Expr, + Left: whereClause.Expr, Right: clauses, } } @@ -188,24 +211,23 @@ func (qs *QuerySplitter) getWhereClause(start, end sqltypes.Value) *sqlparser.Wh } } -func (qs *QuerySplitter) splitBoundaries(pkMinMax *mproto.QueryResult) ([]sqltypes.Value, error) { - boundaries := []sqltypes.Value{} - var err error - // If no min or max values were found, return empty list of boundaries - if len(pkMinMax.Rows) != 1 || pkMinMax.Rows[0][0].IsNull() || pkMinMax.Rows[0][1].IsNull() { - return boundaries, err - } - switch pkMinMax.Fields[0].Type { +func (qs *QuerySplitter) splitBoundaries(columnType int64, pkMinMax *mproto.QueryResult) ([]sqltypes.Value, error) { + switch columnType { case mproto.VT_TINY, mproto.VT_SHORT, mproto.VT_LONG, mproto.VT_LONGLONG, mproto.VT_INT24: - boundaries, err = qs.parseInt(pkMinMax) + return qs.splitBoundariesIntColumn(pkMinMax) case mproto.VT_FLOAT, mproto.VT_DOUBLE: - boundaries, err = qs.parseFloat(pkMinMax) + return qs.splitBoundariesFloatColumn(pkMinMax) + case mproto.VT_VARCHAR, mproto.VT_BIT, mproto.VT_VAR_STRING, mproto.VT_STRING: + return qs.splitBoundariesStringColumn() } - return boundaries, err + return []sqltypes.Value{}, nil } -func (qs *QuerySplitter) parseInt(pkMinMax *mproto.QueryResult) ([]sqltypes.Value, error) { +func (qs *QuerySplitter) splitBoundariesIntColumn(pkMinMax *mproto.QueryResult) ([]sqltypes.Value, error) { boundaries := []sqltypes.Value{} + if pkMinMax == nil || len(pkMinMax.Rows) != 1 || pkMinMax.Rows[0][0].IsNull() || pkMinMax.Rows[0][1].IsNull() { + return boundaries, nil + } minNumeric := sqltypes.MakeNumeric(pkMinMax.Rows[0][0].Raw()) maxNumeric := sqltypes.MakeNumeric(pkMinMax.Rows[0][1].Raw()) if pkMinMax.Rows[0][0].Raw()[0] == '-' { @@ -256,8 +278,11 @@ func (qs *QuerySplitter) parseInt(pkMinMax *mproto.QueryResult) ([]sqltypes.Valu return boundaries, nil } -func (qs *QuerySplitter) parseFloat(pkMinMax *mproto.QueryResult) ([]sqltypes.Value, error) { +func (qs *QuerySplitter) splitBoundariesFloatColumn(pkMinMax *mproto.QueryResult) ([]sqltypes.Value, error) { boundaries := []sqltypes.Value{} + if pkMinMax == nil || len(pkMinMax.Rows) != 1 || pkMinMax.Rows[0][0].IsNull() || pkMinMax.Rows[0][1].IsNull() { + return boundaries, nil + } min, err := strconv.ParseFloat(pkMinMax.Rows[0][0].String(), 64) if err != nil { return nil, err @@ -281,3 +306,27 @@ func (qs *QuerySplitter) parseFloat(pkMinMax *mproto.QueryResult) ([]sqltypes.Va } return boundaries, nil } + +// TODO(shengzhe): support split based on min, max from the string column. +func (qs *QuerySplitter) splitBoundariesStringColumn() ([]sqltypes.Value, error) { + firstRow := int64(0x0) + lastRow := int64(0xFFFFFFFF) + splitRange := lastRow - firstRow + 1 + splitSize := splitRange / int64(qs.splitCount) + qs.rowCount = splitSize + var boundaries []sqltypes.Value + for i := 1; i < qs.splitCount; i++ { + buf := make([]byte, 8) + // encode split point into binaries. + binary.BigEndian.PutUint64(buf, uint64(firstRow+splitSize*int64(i))) + // only converts the lower 4 bytes into hex because the upper 4 bytes are + // always 0x00000000 and mysql does byte comparison from the most significant + // bits. + val, err := sqltypes.BuildValue(buf[4:]) + if err != nil { + return nil, err + } + boundaries = append(boundaries, val) + } + return boundaries, nil +} diff --git a/go/vt/tabletserver/query_splitter_test.go b/go/vt/tabletserver/query_splitter_test.go index 73ba784c0f..3716dba09e 100644 --- a/go/vt/tabletserver/query_splitter_test.go +++ b/go/vt/tabletserver/query_splitter_test.go @@ -1,6 +1,7 @@ package tabletserver import ( + "encoding/binary" "fmt" "reflect" "strings" @@ -102,6 +103,20 @@ func TestValidateQuery(t *testing.T) { t.Errorf("valid query validation failed, got:%v, want:%v", got, want) } + splitter = NewQuerySplitter(query, "id2", 0, schemaInfo) + query.Sql = "select * from test_table where count > :count" + got = splitter.validateQuery() + want = nil + if !reflect.DeepEqual(got, want) { + t.Errorf("valid query validation failed, got:%v, want:%v", got, want) + } + + splitter = NewQuerySplitter(query, "id2", 0, schemaInfo) + query.Sql = "invalid select * from test_table where count > :count" + if err := splitter.validateQuery(); err == nil { + t.Fatalf("validateQuery() = %v, want: nil", err) + } + // column id2 is indexed splitter = NewQuerySplitter(query, "id2", 3, schemaInfo) query.Sql = "select * from test_table where count > :count" @@ -134,10 +149,10 @@ func TestGetWhereClause(t *testing.T) { statement, _ := sqlparser.Parse(sql) splitter.sel, _ = statement.(*sqlparser.Select) splitter.splitColumn = "id" - + bindVars := make(map[string]interface{}) // no boundary case, start = end = nil, should not change the where clause nilValue := sqltypes.Value{} - clause := splitter.getWhereClause(nilValue, nilValue) + clause := splitter.getWhereClause(splitter.sel.Where, bindVars, nilValue, nilValue) want := " where count > :count" got := sqlparser.String(clause) if !reflect.DeepEqual(got, want) { @@ -145,26 +160,45 @@ func TestGetWhereClause(t *testing.T) { } // Set lower bound, should add the lower bound condition to where clause - start, _ := sqltypes.BuildValue(20) - clause = splitter.getWhereClause(start, nilValue) - want = " where count > :count and id >= 20" + startVal := int64(20) + start, _ := sqltypes.BuildValue(startVal) + bindVars = make(map[string]interface{}) + bindVars[":count"] = 300 + clause = splitter.getWhereClause(splitter.sel.Where, bindVars, start, nilValue) + want = " where count > :count and id >= " + startBindVarName got = sqlparser.String(clause) if !reflect.DeepEqual(got, want) { t.Errorf("incorrect where clause, got:%v, want:%v", got, want) } - + v, ok := bindVars[startBindVarName] + if !ok { + t.Fatalf("bind var: %s not found got: nil, want: %v", startBindVarName, startVal) + } + if v != startVal { + t.Fatalf("bind var: %s not found got: %v, want: %v", startBindVarName, v, startVal) + } // Set upper bound, should add the upper bound condition to where clause - end, _ := sqltypes.BuildValue(40) - clause = splitter.getWhereClause(nilValue, end) - want = " where count > :count and id < 40" + endVal := int64(40) + end, _ := sqltypes.BuildValue(endVal) + bindVars = make(map[string]interface{}) + clause = splitter.getWhereClause(splitter.sel.Where, bindVars, nilValue, end) + want = " where count > :count and id < " + endBindVarName got = sqlparser.String(clause) if !reflect.DeepEqual(got, want) { t.Errorf("incorrect where clause, got:%v, want:%v", got, want) } + v, ok = bindVars[endBindVarName] + if !ok { + t.Fatalf("bind var: %s not found got: nil, want: %v", endBindVarName, endVal) + } + if v != endVal { + t.Fatalf("bind var: %s not found got: %v, want: %v", endBindVarName, v, endVal) + } // Set both bounds, should add two conditions to where clause - clause = splitter.getWhereClause(start, end) - want = " where count > :count and id >= 20 and id < 40" + bindVars = make(map[string]interface{}) + clause = splitter.getWhereClause(splitter.sel.Where, bindVars, start, end) + want = fmt.Sprintf(" where count > :count and id >= %s and id < %s", startBindVarName, endBindVarName) got = sqlparser.String(clause) if !reflect.DeepEqual(got, want) { t.Errorf("incorrect where clause, got:%v, want:%v", got, want) @@ -174,22 +208,36 @@ func TestGetWhereClause(t *testing.T) { sql = "select * from test_table" statement, _ = sqlparser.Parse(sql) splitter.sel, _ = statement.(*sqlparser.Select) - + bindVars = make(map[string]interface{}) // no boundary case, start = end = nil should return no where clause - clause = splitter.getWhereClause(nilValue, nilValue) + clause = splitter.getWhereClause(splitter.sel.Where, bindVars, nilValue, nilValue) want = "" got = sqlparser.String(clause) if !reflect.DeepEqual(got, want) { t.Errorf("incorrect where clause for nil ranges, got:%v, want:%v", got, want) } - + bindVars = make(map[string]interface{}) // Set both bounds, should add two conditions to where clause - clause = splitter.getWhereClause(start, end) - want = " where id >= 20 and id < 40" + clause = splitter.getWhereClause(splitter.sel.Where, bindVars, start, end) + want = fmt.Sprintf(" where id >= %s and id < %s", startBindVarName, endBindVarName) got = sqlparser.String(clause) if !reflect.DeepEqual(got, want) { t.Errorf("incorrect where clause, got:%v, want:%v", got, want) } + v, ok = bindVars[startBindVarName] + if !ok { + t.Fatalf("bind var: %s not found got: nil, want: %v", startBindVarName, startVal) + } + if v != startVal { + t.Fatalf("bind var: %s not found got: %v, want: %v", startBindVarName, v, startVal) + } + v, ok = bindVars[endBindVarName] + if !ok { + t.Fatalf("bind var: %s not found got: nil, want: %v", endBindVarName, endVal) + } + if v != endVal { + t.Fatalf("bind var: %s not found got: %v, want: %v", endBindVarName, v, endVal) + } } func TestSplitBoundaries(t *testing.T) { @@ -209,14 +257,14 @@ func TestSplitBoundaries(t *testing.T) { splitter := &QuerySplitter{} splitter.splitCount = 5 - boundaries, err := splitter.splitBoundaries(pkMinMax) + boundaries, err := splitter.splitBoundaries(mproto.VT_LONGLONG, pkMinMax) if err != nil { t.Fatalf("unexpected error: %v", err) } if len(boundaries) != splitter.splitCount-1 { t.Errorf("wrong number of boundaries got: %v, want: %v", len(boundaries), splitter.splitCount-1) } - got, err := splitter.splitBoundaries(pkMinMax) + got, err := splitter.splitBoundaries(mproto.VT_LONGLONG, pkMinMax) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -231,7 +279,7 @@ func TestSplitBoundaries(t *testing.T) { row = []sqltypes.Value{min, max} rows = [][]sqltypes.Value{row} pkMinMax.Rows = rows - got, err = splitter.splitBoundaries(pkMinMax) + got, err = splitter.splitBoundaries(mproto.VT_LONGLONG, pkMinMax) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -250,7 +298,7 @@ func TestSplitBoundaries(t *testing.T) { fields = []mproto.Field{minField, maxField} pkMinMax.Rows = rows pkMinMax.Fields = fields - got, err = splitter.splitBoundaries(pkMinMax) + got, err = splitter.splitBoundaries(mproto.VT_DOUBLE, pkMinMax) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -279,7 +327,7 @@ func TestSplitQuery(t *testing.T) { Type: mproto.VT_LONGLONG, } maxField := mproto.Field{ - Name: "min", + Name: "max", Type: mproto.VT_LONGLONG, } fields := []mproto.Field{minField, maxField} @@ -288,29 +336,143 @@ func TestSplitQuery(t *testing.T) { } // Ensure that empty min max does not cause panic or return any error - splits, err := splitter.split(pkMinMax) + splits, err := splitter.split(mproto.VT_LONGLONG, pkMinMax) if err != nil { t.Errorf("unexpected error while splitting on empty pkMinMax, %s", err) } pkMinMax.Rows = [][]sqltypes.Value{[]sqltypes.Value{min, max}} - splits, err = splitter.split(pkMinMax) + splits, err = splitter.split(mproto.VT_LONGLONG, pkMinMax) if err != nil { t.Fatalf("unexpected error: %v", err) } - got := []string{} + got := []proto.BoundQuery{} for _, split := range splits { if split.RowCount != 100 { t.Errorf("wrong RowCount, got: %v, want: %v", split.RowCount, 100) } - got = append(got, split.Query.Sql) + got = append(got, split.Query) } - want := []string{ - "select * from test_table where count > :count and id < 100", - "select * from test_table where count > :count and id >= 100 and id < 200", - "select * from test_table where count > :count and id >= 200", + want := []proto.BoundQuery{ + { + Sql: "select * from test_table where count > :count and id < " + endBindVarName, + BindVariables: map[string]interface{}{endBindVarName: int64(100)}, + }, + { + Sql: fmt.Sprintf("select * from test_table where count > :count and id >= %s and id < %s", startBindVarName, endBindVarName), + BindVariables: map[string]interface{}{ + startBindVarName: int64(100), + endBindVarName: int64(200), + }, + }, + { + Sql: "select * from test_table where count > :count and id >= " + startBindVarName, + BindVariables: map[string]interface{}{startBindVarName: int64(200)}, + }, } if !reflect.DeepEqual(got, want) { t.Errorf("wrong splits, got: %v, want: %v", got, want) } } + +func TestSplitQueryFractionalColumn(t *testing.T) { + schemaInfo := getSchemaInfo() + query := &proto.BoundQuery{ + Sql: "select * from test_table where count > :count", + } + splitter := NewQuerySplitter(query, "", 3, schemaInfo) + splitter.validateQuery() + min, _ := sqltypes.BuildValue(10.5) + max, _ := sqltypes.BuildValue(490.5) + minField := mproto.Field{ + Name: "min", + Type: mproto.VT_FLOAT, + } + maxField := mproto.Field{ + Name: "max", + Type: mproto.VT_FLOAT, + } + fields := []mproto.Field{minField, maxField} + pkMinMax := &mproto.QueryResult{ + Fields: fields, + Rows: [][]sqltypes.Value{[]sqltypes.Value{min, max}}, + } + + splits, err := splitter.split(mproto.VT_FLOAT, pkMinMax) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + got := []proto.BoundQuery{} + for _, split := range splits { + if split.RowCount != 160 { + t.Errorf("wrong RowCount, got: %v, want: %v", split.RowCount, 160) + } + got = append(got, split.Query) + } + want := []proto.BoundQuery{ + { + Sql: "select * from test_table where count > :count and id < " + endBindVarName, + BindVariables: map[string]interface{}{endBindVarName: 170.5}, + }, + { + Sql: fmt.Sprintf("select * from test_table where count > :count and id >= %s and id < %s", startBindVarName, endBindVarName), + BindVariables: map[string]interface{}{ + startBindVarName: 170.5, + endBindVarName: 330.5, + }, + }, + { + Sql: "select * from test_table where count > :count and id >= " + startBindVarName, + BindVariables: map[string]interface{}{startBindVarName: 330.5}, + }, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("wrong splits, got: %v, want: %v", got, want) + } +} + +func TestSplitQueryStringColumn(t *testing.T) { + schemaInfo := getSchemaInfo() + query := &proto.BoundQuery{ + Sql: "select * from test_table where count > :count", + } + splitter := NewQuerySplitter(query, "", 3, schemaInfo) + splitter.validateQuery() + splits, err := splitter.split(mproto.VT_VAR_STRING, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + got := []proto.BoundQuery{} + for _, split := range splits { + if split.RowCount != 1431655765 { + t.Errorf("wrong RowCount, got: %v, want: %v", split.RowCount, 1431655765) + } + got = append(got, split.Query) + } + want := []proto.BoundQuery{ + { + Sql: "select * from test_table where count > :count and id < " + endBindVarName, + BindVariables: map[string]interface{}{endBindVarName: hexToByteUInt64(0x55555555)[4:]}, + }, + { + Sql: fmt.Sprintf("select * from test_table where count > :count and id >= %s and id < %s", startBindVarName, endBindVarName), + BindVariables: map[string]interface{}{ + startBindVarName: hexToByteUInt64(0x55555555)[4:], + endBindVarName: hexToByteUInt64(0xAAAAAAAA)[4:], + }, + }, + { + Sql: "select * from test_table where count > :count and id >= " + startBindVarName, + BindVariables: map[string]interface{}{startBindVarName: hexToByteUInt64(0xAAAAAAAA)[4:]}, + }, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("wrong splits, got: %v, want: %v", got, want) + } +} + +func hexToByteUInt64(val uint64) []byte { + buf := make([]byte, 8) + binary.BigEndian.PutUint64(buf, val) + return buf +} diff --git a/go/vt/tabletserver/sqlquery.go b/go/vt/tabletserver/sqlquery.go index df4b876d53..dcc07329a8 100644 --- a/go/vt/tabletserver/sqlquery.go +++ b/go/vt/tabletserver/sqlquery.go @@ -539,20 +539,19 @@ func (sq *SqlQuery) SplitQuery(ctx context.Context, target *pb.Target, req *prot logStats: logStats, qe: sq.qe, } - conn, err := qre.getConn(sq.qe.connPool) + columnType, err := getColumnType(qre, splitter.splitColumn, splitter.tableName) if err != nil { return err } - defer conn.Recycle() - // TODO: For fetching MinMax, include where clauses on the - // primary key, if any, in the original query which might give a narrower - // range of split column to work with. - minMaxSQL := fmt.Sprintf("SELECT MIN(%v), MAX(%v) FROM %v", splitter.splitColumn, splitter.splitColumn, splitter.tableName) - splitColumnMinMax, err := qre.execSQL(conn, minMaxSQL, true) - if err != nil { - return err + var pkMinMax *mproto.QueryResult + switch columnType { + case mproto.VT_TINY, mproto.VT_SHORT, mproto.VT_LONG, mproto.VT_LONGLONG, mproto.VT_INT24, mproto.VT_FLOAT, mproto.VT_DOUBLE: + pkMinMax, err = getColumnMinMax(qre, splitter.splitColumn, splitter.tableName) + if err != nil { + return err + } } - reply.Queries, err = splitter.split(splitColumnMinMax) + reply.Queries, err = splitter.split(columnType, pkMinMax) if err != nil { return NewTabletError(ErrFail, "splitQuery: query split error: %s, request: %#v", err, req) } @@ -673,3 +672,30 @@ func withTimeout(ctx context.Context, timeout time.Duration) (context.Context, c } return context.WithTimeout(ctx, timeout) } + +func getColumnType(qre *QueryExecutor, columnName, tableName string) (int64, error) { + conn, err := qre.getConn(qre.qe.connPool) + if err != nil { + return mproto.VT_NULL, err + } + defer conn.Recycle() + query := fmt.Sprintf("SELECT %v FROM %v LIMIT 1", columnName, tableName) + result, err := qre.execSQL(conn, query, true) + if err != nil { + return mproto.VT_NULL, err + } + if result == nil || len(result.Fields) != 1 { + return mproto.VT_NULL, NewTabletError(ErrFail, "failed to get column type for column: %v, invalid result: %v", columnName, result) + } + return result.Fields[0].Type, nil +} + +func getColumnMinMax(qre *QueryExecutor, columnName, tableName string) (*mproto.QueryResult, error) { + conn, err := qre.getConn(qre.qe.connPool) + if err != nil { + return nil, err + } + defer conn.Recycle() + minMaxSQL := fmt.Sprintf("SELECT MIN(%v), MAX(%v) FROM %v", columnName, columnName, tableName) + return qre.execSQL(conn, minMaxSQL, true) +} diff --git a/go/vt/tabletserver/sqlquery_test.go b/go/vt/tabletserver/sqlquery_test.go index 26485f9b39..e6ee4dc03b 100644 --- a/go/vt/tabletserver/sqlquery_test.go +++ b/go/vt/tabletserver/sqlquery_test.go @@ -799,7 +799,17 @@ func TestSqlQuerySplitQuery(t *testing.T) { }, }, }) - + db.AddQuery("SELECT pk FROM test_table LIMIT 1", &mproto.QueryResult{ + Fields: []mproto.Field{ + mproto.Field{Name: "pk", Type: mproto.VT_LONG}, + }, + RowsAffected: 1, + Rows: [][]sqltypes.Value{ + []sqltypes.Value{ + sqltypes.MakeNumeric([]byte("1")), + }, + }, + }) testUtils := newTestUtils() config := testUtils.newQueryServiceConfig() sqlQuery := NewSqlQuery(config) @@ -837,7 +847,30 @@ func TestSqlQuerySplitQuery(t *testing.T) { } func TestSqlQuerySplitQueryInvalidQuery(t *testing.T) { - setUpSqlQueryTest() + db := setUpSqlQueryTest() + db.AddQuery("SELECT MIN(pk), MAX(pk) FROM test_table", &mproto.QueryResult{ + Fields: []mproto.Field{ + mproto.Field{Name: "pk", Type: mproto.VT_LONG}, + }, + RowsAffected: 1, + Rows: [][]sqltypes.Value{ + []sqltypes.Value{ + sqltypes.MakeNumeric([]byte("1")), + sqltypes.MakeNumeric([]byte("100")), + }, + }, + }) + db.AddQuery("SELECT pk FROM test_table LIMIT 1", &mproto.QueryResult{ + Fields: []mproto.Field{ + mproto.Field{Name: "pk", Type: mproto.VT_LONG}, + }, + RowsAffected: 1, + Rows: [][]sqltypes.Value{ + []sqltypes.Value{ + sqltypes.MakeNumeric([]byte("1")), + }, + }, + }) testUtils := newTestUtils() config := testUtils.newQueryServiceConfig() sqlQuery := NewSqlQuery(config) @@ -891,6 +924,17 @@ func TestSqlQuerySplitQueryInvalidMinMax(t *testing.T) { }, }, } + db.AddQuery("SELECT pk FROM test_table LIMIT 1", &mproto.QueryResult{ + Fields: []mproto.Field{ + mproto.Field{Name: "pk", Type: mproto.VT_LONG}, + }, + RowsAffected: 1, + Rows: [][]sqltypes.Value{ + []sqltypes.Value{ + sqltypes.MakeNumeric([]byte("1")), + }, + }, + }) db.AddQuery(pkMinMaxQuery, pkMinMaxQueryResp) config := testUtils.newQueryServiceConfig() From 0c03e61532c44ff51a4e476d709314bd743aec47 Mon Sep 17 00:00:00 2001 From: Shengzhe Yao Date: Fri, 14 Aug 2015 13:03:50 -0700 Subject: [PATCH 17/80] fix lint warnings in sqlquery_test.go --- go/vt/tabletserver/query_splitter.go | 6 +- go/vt/tabletserver/query_splitter_test.go | 26 ++--- go/vt/tabletserver/sqlquery.go | 2 +- go/vt/tabletserver/sqlquery_test.go | 110 +++++++++++----------- 4 files changed, 72 insertions(+), 72 deletions(-) diff --git a/go/vt/tabletserver/query_splitter.go b/go/vt/tabletserver/query_splitter.go index e734d11821..eac4018b68 100644 --- a/go/vt/tabletserver/query_splitter.go +++ b/go/vt/tabletserver/query_splitter.go @@ -201,8 +201,8 @@ func (qs *QuerySplitter) getWhereClause(whereClause *sqlparser.Where, bindVars m } if whereClause != nil { clauses = &sqlparser.AndExpr{ - Left: whereClause.Expr, - Right: clauses, + Left: &sqlparser.ParenBoolExpr{Expr: whereClause.Expr}, + Right: &sqlparser.ParenBoolExpr{Expr: clauses}, } } return &sqlparser.Where{ @@ -310,7 +310,7 @@ func (qs *QuerySplitter) splitBoundariesFloatColumn(pkMinMax *mproto.QueryResult // TODO(shengzhe): support split based on min, max from the string column. func (qs *QuerySplitter) splitBoundariesStringColumn() ([]sqltypes.Value, error) { firstRow := int64(0x0) - lastRow := int64(0xFFFFFFFF) + lastRow := int64(0xFFFFFFFFFFFFFF) splitRange := lastRow - firstRow + 1 splitSize := splitRange / int64(qs.splitCount) qs.rowCount = splitSize diff --git a/go/vt/tabletserver/query_splitter_test.go b/go/vt/tabletserver/query_splitter_test.go index 3716dba09e..5cf1a66849 100644 --- a/go/vt/tabletserver/query_splitter_test.go +++ b/go/vt/tabletserver/query_splitter_test.go @@ -165,7 +165,7 @@ func TestGetWhereClause(t *testing.T) { bindVars = make(map[string]interface{}) bindVars[":count"] = 300 clause = splitter.getWhereClause(splitter.sel.Where, bindVars, start, nilValue) - want = " where count > :count and id >= " + startBindVarName + want = " where (count > :count) and (id >= " + startBindVarName + ")" got = sqlparser.String(clause) if !reflect.DeepEqual(got, want) { t.Errorf("incorrect where clause, got:%v, want:%v", got, want) @@ -182,7 +182,7 @@ func TestGetWhereClause(t *testing.T) { end, _ := sqltypes.BuildValue(endVal) bindVars = make(map[string]interface{}) clause = splitter.getWhereClause(splitter.sel.Where, bindVars, nilValue, end) - want = " where count > :count and id < " + endBindVarName + want = " where (count > :count) and (id < " + endBindVarName + ")" got = sqlparser.String(clause) if !reflect.DeepEqual(got, want) { t.Errorf("incorrect where clause, got:%v, want:%v", got, want) @@ -198,7 +198,7 @@ func TestGetWhereClause(t *testing.T) { // Set both bounds, should add two conditions to where clause bindVars = make(map[string]interface{}) clause = splitter.getWhereClause(splitter.sel.Where, bindVars, start, end) - want = fmt.Sprintf(" where count > :count and id >= %s and id < %s", startBindVarName, endBindVarName) + want = fmt.Sprintf(" where (count > :count) and (id >= %s and id < %s)", startBindVarName, endBindVarName) got = sqlparser.String(clause) if !reflect.DeepEqual(got, want) { t.Errorf("incorrect where clause, got:%v, want:%v", got, want) @@ -355,18 +355,18 @@ func TestSplitQuery(t *testing.T) { } want := []proto.BoundQuery{ { - Sql: "select * from test_table where count > :count and id < " + endBindVarName, + Sql: "select * from test_table where (count > :count) and (id < " + endBindVarName + ")", BindVariables: map[string]interface{}{endBindVarName: int64(100)}, }, { - Sql: fmt.Sprintf("select * from test_table where count > :count and id >= %s and id < %s", startBindVarName, endBindVarName), + Sql: fmt.Sprintf("select * from test_table where (count > :count) and (id >= %s and id < %s)", startBindVarName, endBindVarName), BindVariables: map[string]interface{}{ startBindVarName: int64(100), endBindVarName: int64(200), }, }, { - Sql: "select * from test_table where count > :count and id >= " + startBindVarName, + Sql: "select * from test_table where (count > :count) and (id >= " + startBindVarName + ")", BindVariables: map[string]interface{}{startBindVarName: int64(200)}, }, } @@ -411,18 +411,18 @@ func TestSplitQueryFractionalColumn(t *testing.T) { } want := []proto.BoundQuery{ { - Sql: "select * from test_table where count > :count and id < " + endBindVarName, + Sql: "select * from test_table where (count > :count) and (id < " + endBindVarName + ")", BindVariables: map[string]interface{}{endBindVarName: 170.5}, }, { - Sql: fmt.Sprintf("select * from test_table where count > :count and id >= %s and id < %s", startBindVarName, endBindVarName), + Sql: fmt.Sprintf("select * from test_table where (count > :count) and (id >= %s and id < %s)", startBindVarName, endBindVarName), BindVariables: map[string]interface{}{ startBindVarName: 170.5, endBindVarName: 330.5, }, }, { - Sql: "select * from test_table where count > :count and id >= " + startBindVarName, + Sql: "select * from test_table where (count > :count) and (id >= " + startBindVarName + ")", BindVariables: map[string]interface{}{startBindVarName: 330.5}, }, } @@ -444,25 +444,25 @@ func TestSplitQueryStringColumn(t *testing.T) { } got := []proto.BoundQuery{} for _, split := range splits { - if split.RowCount != 1431655765 { + if split.RowCount != 24019198012642645 { t.Errorf("wrong RowCount, got: %v, want: %v", split.RowCount, 1431655765) } got = append(got, split.Query) } want := []proto.BoundQuery{ { - Sql: "select * from test_table where count > :count and id < " + endBindVarName, + Sql: "select * from test_table where (count > :count) and (id < " + endBindVarName + ")", BindVariables: map[string]interface{}{endBindVarName: hexToByteUInt64(0x55555555)[4:]}, }, { - Sql: fmt.Sprintf("select * from test_table where count > :count and id >= %s and id < %s", startBindVarName, endBindVarName), + Sql: fmt.Sprintf("select * from test_table where (count > :count) and (id >= %s and id < %s)", startBindVarName, endBindVarName), BindVariables: map[string]interface{}{ startBindVarName: hexToByteUInt64(0x55555555)[4:], endBindVarName: hexToByteUInt64(0xAAAAAAAA)[4:], }, }, { - Sql: "select * from test_table where count > :count and id >= " + startBindVarName, + Sql: "select * from test_table where (count > :count) and (id >= " + startBindVarName + ")", BindVariables: map[string]interface{}{startBindVarName: hexToByteUInt64(0xAAAAAAAA)[4:]}, }, } diff --git a/go/vt/tabletserver/sqlquery.go b/go/vt/tabletserver/sqlquery.go index dcc07329a8..9bf862b6bd 100644 --- a/go/vt/tabletserver/sqlquery.go +++ b/go/vt/tabletserver/sqlquery.go @@ -679,7 +679,7 @@ func getColumnType(qre *QueryExecutor, columnName, tableName string) (int64, err return mproto.VT_NULL, err } defer conn.Recycle() - query := fmt.Sprintf("SELECT %v FROM %v LIMIT 1", columnName, tableName) + query := fmt.Sprintf("SELECT %v FROM %v LIMIT 0", columnName, tableName) result, err := qre.execSQL(conn, query, true) if err != nil { return mproto.VT_NULL, err diff --git a/go/vt/tabletserver/sqlquery_test.go b/go/vt/tabletserver/sqlquery_test.go index e6ee4dc03b..be28157ef6 100644 --- a/go/vt/tabletserver/sqlquery_test.go +++ b/go/vt/tabletserver/sqlquery_test.go @@ -19,28 +19,28 @@ import ( ) func TestSqlQueryAllowQueriesFailBadConn(t *testing.T) { - db := setUpSqlQueryTest() + db := setUpSQLQueryTest() db.EnableConnFail() testUtils := newTestUtils() config := testUtils.newQueryServiceConfig() sqlQuery := NewSqlQuery(config) - checkSqlQueryState(t, sqlQuery, "NOT_SERVING") + checkSQLQueryState(t, sqlQuery, "NOT_SERVING") dbconfigs := testUtils.newDBConfigs() err := sqlQuery.allowQueries(nil, &dbconfigs, []SchemaOverride{}, testUtils.newMysqld(&dbconfigs)) if err == nil { t.Fatalf("SqlQuery.allowQueries should fail") } - checkSqlQueryState(t, sqlQuery, "NOT_SERVING") + checkSQLQueryState(t, sqlQuery, "NOT_SERVING") } func TestSqlQueryAllowQueriesFailStrictModeConflictWithRowCache(t *testing.T) { - setUpSqlQueryTest() + setUpSQLQueryTest() testUtils := newTestUtils() config := testUtils.newQueryServiceConfig() // disable strict mode config.StrictMode = false sqlQuery := NewSqlQuery(config) - checkSqlQueryState(t, sqlQuery, "NOT_SERVING") + checkSQLQueryState(t, sqlQuery, "NOT_SERVING") dbconfigs := testUtils.newDBConfigs() // enable rowcache dbconfigs.App.EnableRowcache = true @@ -48,15 +48,15 @@ func TestSqlQueryAllowQueriesFailStrictModeConflictWithRowCache(t *testing.T) { if err == nil { t.Fatalf("SqlQuery.allowQueries should fail because strict mode is disabled while rowcache is enabled.") } - checkSqlQueryState(t, sqlQuery, "NOT_SERVING") + checkSQLQueryState(t, sqlQuery, "NOT_SERVING") } func TestSqlQueryAllowQueries(t *testing.T) { - setUpSqlQueryTest() + setUpSQLQueryTest() testUtils := newTestUtils() config := testUtils.newQueryServiceConfig() sqlQuery := NewSqlQuery(config) - checkSqlQueryState(t, sqlQuery, "NOT_SERVING") + checkSQLQueryState(t, sqlQuery, "NOT_SERVING") dbconfigs := testUtils.newDBConfigs() sqlQuery.setState(StateServing) err := sqlQuery.allowQueries(nil, &dbconfigs, []SchemaOverride{}, testUtils.newMysqld(&dbconfigs)) @@ -73,7 +73,7 @@ func TestSqlQueryAllowQueries(t *testing.T) { } func TestSqlQueryCheckMysql(t *testing.T) { - setUpSqlQueryTest() + setUpSQLQueryTest() testUtils := newTestUtils() config := testUtils.newQueryServiceConfig() sqlQuery := NewSqlQuery(config) @@ -89,7 +89,7 @@ func TestSqlQueryCheckMysql(t *testing.T) { } func TestSqlQueryCheckMysqlFailInvalidConn(t *testing.T) { - db := setUpSqlQueryTest() + db := setUpSQLQueryTest() testUtils := newTestUtils() config := testUtils.newQueryServiceConfig() sqlQuery := NewSqlQuery(config) @@ -107,7 +107,7 @@ func TestSqlQueryCheckMysqlFailInvalidConn(t *testing.T) { } func TestSqlQueryCheckMysqlFailUninitializedQueryEngine(t *testing.T) { - setUpSqlQueryTest() + setUpSQLQueryTest() testUtils := newTestUtils() config := testUtils.newQueryServiceConfig() sqlQuery := NewSqlQuery(config) @@ -126,7 +126,7 @@ func TestSqlQueryCheckMysqlFailUninitializedQueryEngine(t *testing.T) { } func TestSqlQueryCheckMysqlInNotServingState(t *testing.T) { - setUpSqlQueryTest() + setUpSQLQueryTest() testUtils := newTestUtils() config := testUtils.newQueryServiceConfig() config.EnablePublishStats = true @@ -151,7 +151,7 @@ func TestSqlQueryCheckMysqlInNotServingState(t *testing.T) { } func TestSqlQueryGetSessionId(t *testing.T) { - setUpSqlQueryTest() + setUpSQLQueryTest() testUtils := newTestUtils() config := testUtils.newQueryServiceConfig() sqlQuery := NewSqlQuery(config) @@ -196,7 +196,7 @@ func TestSqlQueryGetSessionId(t *testing.T) { } func TestSqlQueryCommandFailUnMatchedSessionId(t *testing.T) { - setUpSqlQueryTest() + setUpSQLQueryTest() testUtils := newTestUtils() config := testUtils.newQueryServiceConfig() sqlQuery := NewSqlQuery(config) @@ -286,17 +286,17 @@ func TestSqlQueryCommandFailUnMatchedSessionId(t *testing.T) { } func TestSqlQueryCommitTransaciton(t *testing.T) { - db := setUpSqlQueryTest() + db := setUpSQLQueryTest() testUtils := newTestUtils() // sql that will be executed in this test - executeSql := "select * from test_table limit 1000" - executeSqlResult := &mproto.QueryResult{ + executeSQL := "select * from test_table limit 1000" + executeSQLResult := &mproto.QueryResult{ RowsAffected: 1, Rows: [][]sqltypes.Value{ []sqltypes.Value{sqltypes.MakeString([]byte("row01"))}, }, } - db.AddQuery(executeSql, executeSqlResult) + db.AddQuery(executeSQL, executeSQLResult) config := testUtils.newQueryServiceConfig() sqlQuery := NewSqlQuery(config) dbconfigs := testUtils.newDBConfigs() @@ -316,7 +316,7 @@ func TestSqlQueryCommitTransaciton(t *testing.T) { } session.TransactionId = txInfo.TransactionId query := proto.Query{ - Sql: executeSql, + Sql: executeSQL, BindVariables: nil, SessionId: session.SessionId, TransactionId: session.TransactionId, @@ -331,17 +331,17 @@ func TestSqlQueryCommitTransaciton(t *testing.T) { } func TestSqlQueryRollback(t *testing.T) { - db := setUpSqlQueryTest() + db := setUpSQLQueryTest() testUtils := newTestUtils() // sql that will be executed in this test - executeSql := "select * from test_table limit 1000" - executeSqlResult := &mproto.QueryResult{ + executeSQL := "select * from test_table limit 1000" + executeSQLResult := &mproto.QueryResult{ RowsAffected: 1, Rows: [][]sqltypes.Value{ []sqltypes.Value{sqltypes.MakeString([]byte("row01"))}, }, } - db.AddQuery(executeSql, executeSqlResult) + db.AddQuery(executeSQL, executeSQLResult) config := testUtils.newQueryServiceConfig() sqlQuery := NewSqlQuery(config) dbconfigs := testUtils.newDBConfigs() @@ -361,7 +361,7 @@ func TestSqlQueryRollback(t *testing.T) { } session.TransactionId = txInfo.TransactionId query := proto.Query{ - Sql: executeSql, + Sql: executeSQL, BindVariables: nil, SessionId: session.SessionId, TransactionId: session.TransactionId, @@ -376,17 +376,17 @@ func TestSqlQueryRollback(t *testing.T) { } func TestSqlQueryStreamExecute(t *testing.T) { - db := setUpSqlQueryTest() + db := setUpSQLQueryTest() testUtils := newTestUtils() // sql that will be executed in this test - executeSql := "select * from test_table limit 1000" - executeSqlResult := &mproto.QueryResult{ + executeSQL := "select * from test_table limit 1000" + executeSQLResult := &mproto.QueryResult{ RowsAffected: 1, Rows: [][]sqltypes.Value{ []sqltypes.Value{sqltypes.MakeString([]byte("row01"))}, }, } - db.AddQuery(executeSql, executeSqlResult) + db.AddQuery(executeSQL, executeSQLResult) config := testUtils.newQueryServiceConfig() sqlQuery := NewSqlQuery(config) @@ -407,7 +407,7 @@ func TestSqlQueryStreamExecute(t *testing.T) { } session.TransactionId = txInfo.TransactionId query := proto.Query{ - Sql: executeSql, + Sql: executeSQL, BindVariables: nil, SessionId: session.SessionId, TransactionId: session.TransactionId, @@ -427,14 +427,14 @@ func TestSqlQueryStreamExecute(t *testing.T) { } func TestSqlQueryExecuteBatch(t *testing.T) { - db := setUpSqlQueryTest() + db := setUpSQLQueryTest() testUtils := newTestUtils() sql := "insert into test_table values (1, 2)" sqlResult := &mproto.QueryResult{} - expanedSql := "insert into test_table values (1, 2) /* _stream test_table (pk ) (1 ); */" + expanedSQL := "insert into test_table values (1, 2) /* _stream test_table (pk ) (1 ); */" db.AddQuery(sql, sqlResult) - db.AddQuery(expanedSql, sqlResult) + db.AddQuery(expanedSQL, sqlResult) config := testUtils.newQueryServiceConfig() sqlQuery := NewSqlQuery(config) dbconfigs := testUtils.newDBConfigs() @@ -469,7 +469,7 @@ func TestSqlQueryExecuteBatch(t *testing.T) { } func TestSqlQueryExecuteBatchFailEmptyQueryList(t *testing.T) { - setUpSqlQueryTest() + setUpSQLQueryTest() testUtils := newTestUtils() config := testUtils.newQueryServiceConfig() sqlQuery := NewSqlQuery(config) @@ -493,7 +493,7 @@ func TestSqlQueryExecuteBatchFailEmptyQueryList(t *testing.T) { } func TestSqlQueryExecuteBatchFailAsTransaction(t *testing.T) { - setUpSqlQueryTest() + setUpSQLQueryTest() testUtils := newTestUtils() config := testUtils.newQueryServiceConfig() sqlQuery := NewSqlQuery(config) @@ -524,7 +524,7 @@ func TestSqlQueryExecuteBatchFailAsTransaction(t *testing.T) { } func TestSqlQueryExecuteBatchBeginFail(t *testing.T) { - db := setUpSqlQueryTest() + db := setUpSQLQueryTest() testUtils := newTestUtils() // make "begin" query fail db.AddRejectedQuery("begin") @@ -558,7 +558,7 @@ func TestSqlQueryExecuteBatchBeginFail(t *testing.T) { } func TestSqlQueryExecuteBatchCommitFail(t *testing.T) { - db := setUpSqlQueryTest() + db := setUpSQLQueryTest() testUtils := newTestUtils() // make "commit" query fail db.AddRejectedQuery("commit") @@ -597,18 +597,18 @@ func TestSqlQueryExecuteBatchCommitFail(t *testing.T) { } func TestSqlQueryExecuteBatchSqlExecFailInTransaction(t *testing.T) { - db := setUpSqlQueryTest() + db := setUpSQLQueryTest() testUtils := newTestUtils() sql := "insert into test_table values (1, 2)" sqlResult := &mproto.QueryResult{} - expanedSql := "insert into test_table values (1, 2) /* _stream test_table (pk ) (1 ); */" + expanedSQL := "insert into test_table values (1, 2) /* _stream test_table (pk ) (1 ); */" db.AddQuery(sql, sqlResult) - db.AddQuery(expanedSql, sqlResult) + db.AddQuery(expanedSQL, sqlResult) // make this query fail db.AddRejectedQuery(sql) - db.AddRejectedQuery(expanedSql) + db.AddRejectedQuery(expanedSQL) config := testUtils.newQueryServiceConfig() sqlQuery := NewSqlQuery(config) @@ -652,14 +652,14 @@ func TestSqlQueryExecuteBatchSqlExecFailInTransaction(t *testing.T) { } func TestSqlQueryExecuteBatchSqlSucceedInTransaction(t *testing.T) { - db := setUpSqlQueryTest() + db := setUpSQLQueryTest() testUtils := newTestUtils() sql := "insert into test_table values (1, 2)" sqlResult := &mproto.QueryResult{} - expanedSql := "insert into test_table values (1, 2) /* _stream test_table (pk ) (1 ); */" + expanedSQL := "insert into test_table values (1, 2) /* _stream test_table (pk ) (1 ); */" db.AddQuery(sql, sqlResult) - db.AddQuery(expanedSql, sqlResult) + db.AddQuery(expanedSQL, sqlResult) // cause execution error for this particular sql query db.AddRejectedQuery(sql) @@ -695,7 +695,7 @@ func TestSqlQueryExecuteBatchSqlSucceedInTransaction(t *testing.T) { } func TestSqlQueryExecuteBatchCallCommitWithoutABegin(t *testing.T) { - setUpSqlQueryTest() + setUpSQLQueryTest() testUtils := newTestUtils() config := testUtils.newQueryServiceConfig() sqlQuery := NewSqlQuery(config) @@ -727,14 +727,14 @@ func TestSqlQueryExecuteBatchCallCommitWithoutABegin(t *testing.T) { } func TestExecuteBatchNestedTransaction(t *testing.T) { - db := setUpSqlQueryTest() + db := setUpSQLQueryTest() testUtils := newTestUtils() sql := "insert into test_table values (1, 2)" sqlResult := &mproto.QueryResult{} - expanedSql := "insert into test_table values (1, 2) /* _stream test_table (pk ) (1 ); */" + expanedSQL := "insert into test_table values (1, 2) /* _stream test_table (pk ) (1 ); */" db.AddQuery(sql, sqlResult) - db.AddQuery(expanedSql, sqlResult) + db.AddQuery(expanedSQL, sqlResult) config := testUtils.newQueryServiceConfig() sqlQuery := NewSqlQuery(config) dbconfigs := testUtils.newDBConfigs() @@ -786,7 +786,7 @@ func TestExecuteBatchNestedTransaction(t *testing.T) { } func TestSqlQuerySplitQuery(t *testing.T) { - db := setUpSqlQueryTest() + db := setUpSQLQueryTest() db.AddQuery("SELECT MIN(pk), MAX(pk) FROM test_table", &mproto.QueryResult{ Fields: []mproto.Field{ mproto.Field{Name: "pk", Type: mproto.VT_LONG}, @@ -799,7 +799,7 @@ func TestSqlQuerySplitQuery(t *testing.T) { }, }, }) - db.AddQuery("SELECT pk FROM test_table LIMIT 1", &mproto.QueryResult{ + db.AddQuery("SELECT pk FROM test_table LIMIT 0", &mproto.QueryResult{ Fields: []mproto.Field{ mproto.Field{Name: "pk", Type: mproto.VT_LONG}, }, @@ -847,7 +847,7 @@ func TestSqlQuerySplitQuery(t *testing.T) { } func TestSqlQuerySplitQueryInvalidQuery(t *testing.T) { - db := setUpSqlQueryTest() + db := setUpSQLQueryTest() db.AddQuery("SELECT MIN(pk), MAX(pk) FROM test_table", &mproto.QueryResult{ Fields: []mproto.Field{ mproto.Field{Name: "pk", Type: mproto.VT_LONG}, @@ -860,7 +860,7 @@ func TestSqlQuerySplitQueryInvalidQuery(t *testing.T) { }, }, }) - db.AddQuery("SELECT pk FROM test_table LIMIT 1", &mproto.QueryResult{ + db.AddQuery("SELECT pk FROM test_table LIMIT 0", &mproto.QueryResult{ Fields: []mproto.Field{ mproto.Field{Name: "pk", Type: mproto.VT_LONG}, }, @@ -908,7 +908,7 @@ func TestSqlQuerySplitQueryInvalidQuery(t *testing.T) { } func TestSqlQuerySplitQueryInvalidMinMax(t *testing.T) { - db := setUpSqlQueryTest() + db := setUpSQLQueryTest() testUtils := newTestUtils() pkMinMaxQuery := "SELECT MIN(pk), MAX(pk) FROM test_table" pkMinMaxQueryResp := &mproto.QueryResult{ @@ -924,7 +924,7 @@ func TestSqlQuerySplitQueryInvalidMinMax(t *testing.T) { }, }, } - db.AddQuery("SELECT pk FROM test_table LIMIT 1", &mproto.QueryResult{ + db.AddQuery("SELECT pk FROM test_table LIMIT 0", &mproto.QueryResult{ Fields: []mproto.Field{ mproto.Field{Name: "pk", Type: mproto.VT_LONG}, }, @@ -1055,7 +1055,7 @@ func TestTerseErrors2(t *testing.T) { }) } -func setUpSqlQueryTest() *fakesqldb.DB { +func setUpSQLQueryTest() *fakesqldb.DB { db := fakesqldb.Register() for query, result := range getSupportedQueries() { db.AddQuery(query, result) @@ -1063,7 +1063,7 @@ func setUpSqlQueryTest() *fakesqldb.DB { return db } -func checkSqlQueryState(t *testing.T, sqlQuery *SqlQuery, expectState string) { +func checkSQLQueryState(t *testing.T, sqlQuery *SqlQuery, expectState string) { if sqlQuery.GetState() != expectState { t.Fatalf("sqlquery should in state: %s, but get state: %s", expectState, sqlQuery.GetState()) } From e20dabd73f63f88df426f584a1975ea6bba2e5e5 Mon Sep 17 00:00:00 2001 From: Anthony Yeh Date: Mon, 17 Aug 2015 11:23:54 -0700 Subject: [PATCH 18/80] Remove obsolete go/db package. This interface is no longer imported or implemented anywhere. It was only used by vtclient2, which has been removed. --- go/db/db.go | 64 ----------------------------------------------------- 1 file changed, 64 deletions(-) delete mode 100644 go/db/db.go diff --git a/go/db/db.go b/go/db/db.go deleted file mode 100644 index 9ed6cb711a..0000000000 --- a/go/db/db.go +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2012, Google Inc. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package db defines an alternate (and simplified) -// db api compared to go's database/sql. -package db - -import ( - "fmt" -) - -// Driver is the interface that must be implemented by a database driver. -type Driver interface { - // Open returns a new connection to the database. - // The name is a string in a driver-specific format. - // The returned connection should only be used by one - // goroutine at a time. - Open(name string) (Conn, error) -} - -var drivers = make(map[string]Driver) - -// Register makes a database driver available by the provided name. -func Register(name string, driver Driver) { - drivers[name] = driver -} - -// Open opens a database specified by its database driver name -// and a driver-specific data source name, usually consisting -// of at least a database name and connection information. -func Open(name, path string) (Conn, error) { - d := drivers[name] - if d == nil { - return nil, fmt.Errorf("Driver %s not found", name) - } - return d.Open(path) -} - -// Conn is a connection to a database. It should not be used -// concurrently by multiple goroutines. -type Conn interface { - Exec(query string, args map[string]interface{}) (Result, error) - Begin() (Tx, error) - Close() error -} - -// Tx is a transaction. -type Tx interface { - Commit() error - Rollback() error -} - -// Result is an iterator over an executed query's results. -// It is also used to query for the results of a DML, in which -// case the iterator functions are not applicable. -type Result interface { - LastInsertId() (int64, error) - RowsAffected() (int64, error) - Columns() []string - Next() []interface{} - Err() error - Close() error -} From 1251842c0484232567fe0a4990dde59c78ada8dc Mon Sep 17 00:00:00 2001 From: Alain Jobart Date: Mon, 17 Aug 2015 11:34:25 -0700 Subject: [PATCH 19/80] Fixing rowcache_invalidator.py to not depend on zkocc nor sleep. --- test/rowcache_invalidator.py | 70 +++++++++++++++++++----------------- 1 file changed, 37 insertions(+), 33 deletions(-) diff --git a/test/rowcache_invalidator.py b/test/rowcache_invalidator.py index 32fe814e6c..33c323188e 100755 --- a/test/rowcache_invalidator.py +++ b/test/rowcache_invalidator.py @@ -99,7 +99,8 @@ class RowCacheInvalidator(unittest.TestCase): def perform_insert(self, count): for i in xrange(count): - self._exec_vt_txn("insert into vt_insert_test (msg) values ('test %s')" % i) + self._exec_vt_txn("insert into vt_insert_test (msg) values ('test %s')" % + i) def perform_delete(self): self._exec_vt_txn('delete from vt_insert_test') @@ -142,22 +143,29 @@ class RowCacheInvalidator(unittest.TestCase): self.assertEqual(stats_dict['Hits'] - hits, 1, "This should have hit the cache") + def _wait_for_value(self, expected_result): + timeout = 10 + while True: + result = self._exec_replica_query( + 'select * from vt_insert_test where id = 1000000') + if result == expected_result: + return + timeout = utils.wait_step('replica rowcache updated, got %s expected %s' % + (str(result), str(expected_result)), timeout, + sleep_time=0.1) + def test_outofband_statements(self): start = self.replica_vars()['InternalErrors'].get('Invalidation', 0) - self._exec_vt_txn("insert into vt_insert_test (id, msg) values (1000000, 'start')") - self._wait_for_replica() - time.sleep(1.0) # Test update statement - result = self._exec_replica_query('select * from vt_insert_test where id = 1000000') - self.assertEqual(result, [['1000000', 'start']]) + self._exec_vt_txn("insert into vt_insert_test (id, msg) values (1000000, 'start')") + self._wait_for_replica() + self._wait_for_value([['1000000', 'start']]) utils.mysql_write_query(master_tablet.tablet_uid, 'vt_test_keyspace', "update vt_insert_test set msg = 'foo' where id = 1000000") self._wait_for_replica() - time.sleep(1.0) - result = self._exec_replica_query('select * from vt_insert_test where id = 1000000') - self.assertEqual(result, [['1000000', 'foo']]) + self._wait_for_value([['1000000', 'foo']]) end1 = self.replica_vars()['InternalErrors'].get('Invalidation', 0) self.assertEqual(start, end1) @@ -166,9 +174,7 @@ class RowCacheInvalidator(unittest.TestCase): 'vt_test_keyspace', 'delete from vt_insert_test where id = 1000000') self._wait_for_replica() - time.sleep(1.0) - result = self._exec_replica_query('select * from vt_insert_test where id = 1000000') - self.assertEqual(result, []) + self._wait_for_value([]) end2 = self.replica_vars()['InternalErrors'].get('Invalidation', 0) self.assertEqual(end1, end2) @@ -177,9 +183,7 @@ class RowCacheInvalidator(unittest.TestCase): 'vt_test_keyspace', "insert into vt_insert_test (id, msg) values(1000000, 'bar')") self._wait_for_replica() - time.sleep(1.0) - result = self._exec_replica_query('select * from vt_insert_test where id = 1000000') - self.assertEqual(result, [['1000000', 'bar']]) + self._wait_for_value([['1000000', 'bar']]) end3 = self.replica_vars()['InternalErrors'].get('Invalidation', 0) self.assertEqual(end2, end3) @@ -188,8 +192,13 @@ class RowCacheInvalidator(unittest.TestCase): 'vt_test_keyspace', 'truncate table vt_insert_test') self._wait_for_replica() - time.sleep(1.0) - end4 = self.replica_vars()['InternalErrors'].get('Invalidation', 0) + timeout = 10 + while True: + end4 = self.replica_vars()['InternalErrors'].get('Invalidation', 0) + if end4 == end3+1: + break + timeout = utils.wait_step('invalidation errors, got %d expecting %d' % + (end4, end3+1), timeout, sleep_time=0.1) self.assertEqual(end4, end3+1) def test_stop_replication(self): @@ -205,15 +214,13 @@ class RowCacheInvalidator(unittest.TestCase): self._wait_for_replica() # wait until the slave processed all data - for timeout in xrange(300): - time.sleep(0.1) + timeout = 30 + while True: inv_count1 = self.replica_stats()['Totals']['Invalidations'] - logging.debug('Got %d invalidations' % inv_count1) if inv_count1 == 100: break - inv_count1 = self.replica_stats()['Totals']['Invalidations'] - self.assertEqual(inv_count1, 100, - 'Unexpected number of invalidations: %d' % inv_count1) + timeout = utils.wait_step('invalidation count, got %d expecting %d' % + (inv_count1, 100), timeout, sleep_time=0.1) # stop replication insert more data, restart replication replica_tablet.mquery('vt_test_keyspace', 'stop slave') @@ -223,15 +230,13 @@ class RowCacheInvalidator(unittest.TestCase): self._wait_for_replica() # wait until the slave processed all data - for timeout in xrange(300): - time.sleep(0.1) + timeout = 30 + while True: inv_count2 = self.replica_stats()['Totals']['Invalidations'] - logging.debug('Got %d invalidations' % inv_count2) if inv_count2 == 200: break - inv_count2 = self.replica_stats()['Totals']['Invalidations'] - self.assertEqual(inv_count2, 200, 'Unexpected number of invalidations: %d' % - inv_count2) + timeout = utils.wait_step('invalidation count, got %d expecting %d' % + (inv_count2, 200), timeout, sleep_time=0.1) # check and display some stats invalidatorStats = self.replica_vars() @@ -268,13 +273,12 @@ class RowCacheInvalidator(unittest.TestCase): utils.run_vtctl(['ChangeSlaveType', replica_tablet.tablet_alias, 'spare']) # wait until it's stopped - for timeout in xrange(300): - time.sleep(0.1) + timeout = 30 + while True: invStats_after = self.replica_vars() - logging.debug('Got state %s' % - invStats_after['RowcacheInvalidatorState']) if invStats_after['RowcacheInvalidatorState'] == 'Stopped': break + timeout = utils.wait_step('RowcacheInvalidatorState, got %s expecting Stopped' % invStats_after['RowcacheInvalidatorState'], timeout, sleep_time=0.1) # check all data is right inv_after = self.replica_stats()['Totals']['Invalidations'] From 08d6933e2347acb8191b5a7119f44b141f234e71 Mon Sep 17 00:00:00 2001 From: Alain Jobart Date: Mon, 17 Aug 2015 11:59:29 -0700 Subject: [PATCH 20/80] Fixing this test after my hook change. --- test/tabletmanager.py | 84 ++++++++++++++++++++++--------------------- 1 file changed, 43 insertions(+), 41 deletions(-) diff --git a/test/tabletmanager.py b/test/tabletmanager.py index 20031965c8..79e2f4702e 100755 --- a/test/tabletmanager.py +++ b/test/tabletmanager.py @@ -242,22 +242,23 @@ class TestTabletManager(unittest.TestCase): # TODO(szopa): Test that non-authenticated queries do not pass # through (when we get to that point). - def _check_string_in_hook_result(self, text, expected): - if isinstance(expected, basestring): - expected = [expected] - for exp in expected: - if exp in text: - return - logging.warning("ExecuteHook output:\n%s", text) - self.fail("ExecuteHook returned unexpected result, no string: '" + "', '".join(expected) + "'") - - def _run_hook(self, params, expectedStrings): - out, err = utils.run_vtctl(['--alsologtostderr', 'ExecuteHook', - tablet_62344.tablet_alias] + params, - mode=utils.VTCTL_VTCTL, trap_output=True, - raise_on_error=False) - for expected in expectedStrings: - self._check_string_in_hook_result(err, expected) + def _run_hook(self, params, expectedStatus, expectedStdout, expectedStderr): + hr = utils.run_vtctl_json(['ExecuteHook', tablet_62344.tablet_alias] + + params) + self.assertEqual(hr['ExitStatus'], expectedStatus) + if isinstance(expectedStdout, basestring): + if expectedStdout[-1:] == '%': + self.assertEqual(hr['Stdout'][:len(expectedStdout)-1], expectedStdout[:len(expectedStdout)-1]) + else: + self.assertEqual(hr['Stdout'], expectedStdout) + else: + found = False + for exp in expectedStdout: + if hr['Stdout'] == exp: + found = True + break + if not found: + self.assertFail('cannot find expected %s in %s' % (str(expectedStdout), hr['Stdout'])) def test_hook(self): utils.run_vtctl(['CreateKeyspace', 'test_keyspace']) @@ -268,39 +269,40 @@ class TestTabletManager(unittest.TestCase): tablet_62344.init_tablet('master', 'test_keyspace', '0', start=True) # test a regular program works - self._run_hook(['test.sh', '--flag1', '--param1=hello'], [ - '"ExitStatus": 0', - ['"Stdout": "TABLET_ALIAS: test_nj-0000062344\\nPARAM: --flag1\\nPARAM: --param1=hello\\n"', - '"Stdout": "TABLET_ALIAS: test_nj-0000062344\\nPARAM: --param1=hello\\nPARAM: --flag1\\n"', - ], - '"Stderr": ""', - ]) + self._run_hook(['test.sh', '--flag1', '--param1=hello'], 0, + ['TABLET_ALIAS: test_nj-0000062344\n' + + 'PARAM: --flag1\n' + + 'PARAM: --param1=hello\n', + 'TABLET_ALIAS: test_nj-0000062344\n' + + 'PARAM: --param1=hello\n' + + 'PARAM: --flag1\n'], + '') # test stderr output - self._run_hook(['test.sh', '--to-stderr'], [ - '"ExitStatus": 0', - '"Stdout": "TABLET_ALIAS: test_nj-0000062344\\nPARAM: --to-stderr\\n"', - '"Stderr": "ERR: --to-stderr\\n"', - ]) + self._run_hook(['test.sh', '--to-stderr'], 0, + 'TABLET_ALIAS: test_nj-0000062344\n' + + 'PARAM: --to-stderr\n', + 'ERR: --to-stderr\n') # test commands that fail - self._run_hook(['test.sh', '--exit-error'], [ - '"ExitStatus": 1', - '"Stdout": "TABLET_ALIAS: test_nj-0000062344\\nPARAM: --exit-error\\n"', - '"Stderr": "ERROR: exit status 1\\n"', - ]) + self._run_hook(['test.sh', '--exit-error'], 1, + 'TABLET_ALIAS: test_nj-0000062344\n' + + 'PARAM: --exit-error\n', + 'ERROR: exit status 1\n') # test hook that is not present - self._run_hook(['not_here.sh'], [ - '"ExitStatus": -1', - '"Stdout": "Skipping missing hook: /', # cannot go further, local path - '"Stderr": ""', - ]) + self._run_hook(['not_here.sh'], -1, + 'Skipping missing hook: /%', # cannot go further, local path + '') # test hook with invalid name - self._run_hook(['/bin/ls'], [ - "action failed: ExecuteHook hook name cannot have a '/' in it", - ]) + _, err = utils.run_vtctl(['--alsologtostderr', 'ExecuteHook', + tablet_62344.tablet_alias, + '/bin/ls'], + mode=utils.VTCTL_VTCTL, trap_output=True, + raise_on_error=False) + expected = "action failed: ExecuteHook hook name cannot have a '/' in it" + self.assertIn(expected, err) tablet_62344.kill_vttablet() From 4ebc1ed16b7c45e079e442b76e01663ce1fddf74 Mon Sep 17 00:00:00 2001 From: Michael Berlin Date: Mon, 17 Aug 2015 12:58:56 -0700 Subject: [PATCH 21/80] proto: Change field indexes in query.proto. This is safe because they are not used anywhere else. --- proto/query.proto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/proto/query.proto b/proto/query.proto index 42591d647d..f393862789 100644 --- a/proto/query.proto +++ b/proto/query.proto @@ -294,10 +294,10 @@ message RealtimeStats { // master of an ongoing filtered replication only. // It is used to find out how far the receiving master lags behind the // source shard. - int64 filtered_replication_synced_until_timestamp = 4; + int64 filtered_replication_synced_until_timestamp = 3; // cpu_usage is used for load-based balancing - double cpu_usage = 3; + double cpu_usage = 4; } // StreamHealthResponse is streamed by StreamHealth on a regular basis From fe3dde842d030da5e13f74762af61de3c2ef2b1d Mon Sep 17 00:00:00 2001 From: Michael Berlin Date: Mon, 17 Aug 2015 13:09:47 -0700 Subject: [PATCH 22/80] test: Replace assertTrue(member in collection) constructs with assertIn(member, collection). assertIn has a nicer error formatting. --- test/framework.py | 2 +- test/queryservice_tests/stream_tests.py | 2 +- test/resharding.py | 18 +++++++++--------- test/update_stream.py | 4 ++-- test/vtgatev2_test.py | 2 +- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/test/framework.py b/test/framework.py index d635b36a8b..62dd555391 100644 --- a/test/framework.py +++ b/test/framework.py @@ -16,7 +16,7 @@ class TestCase(unittest.TestCase): cls.env = env def assertContains(self, b, a): - self.assertTrue(a in b, "%r not found in %r" % (a, b)) + self.assertIn(a, b) class MultiDict(dict): def __getattr__(self, name): diff --git a/test/queryservice_tests/stream_tests.py b/test/queryservice_tests/stream_tests.py index d0787604b6..6a69f17f96 100644 --- a/test/queryservice_tests/stream_tests.py +++ b/test/queryservice_tests/stream_tests.py @@ -114,7 +114,7 @@ class TestStream(framework.TestCase): cu.fetchall() errMsg1 = "error: the query was killed either because it timed out or was canceled: Lost connectioy to MySQL server during query (errno 2013)" errMsg2 = "error: Query execution was interrupted (errno 1317)" - self.assertTrue(cm.exception not in (errMsg1, errMsg2), "did not raise interruption error: %s" % str(cm.exception)) + self.assertNotIn(cm.exception, (errMsg1, errMsg2), "did not raise interruption error: %s" % str(cm.exception)) cu.close() except Exception, e: self.fail("Failed with error %s %s" % (str(e), traceback.print_exc())) diff --git a/test/resharding.py b/test/resharding.py index 499dc66a0a..764b6f08b6 100755 --- a/test/resharding.py +++ b/test/resharding.py @@ -379,19 +379,19 @@ primary key (name) def _check_binlog_server_vars(self, tablet): v = utils.get_vars(tablet.port) - self.assertTrue('UpdateStreamKeyRangeStatements' in v) - self.assertTrue('UpdateStreamKeyRangeTransactions' in v) + self.assertIn('UpdateStreamKeyRangeStatements', v) + self.assertIn('UpdateStreamKeyRangeTransactions', v) def _check_binlog_player_vars(self, tablet, seconds_behind_master_max = 0): v = utils.get_vars(tablet.port) - self.assertTrue('BinlogPlayerMapSize' in v) - self.assertTrue('BinlogPlayerSecondsBehindMaster' in v) - self.assertTrue('BinlogPlayerSecondsBehindMasterMap' in v) - self.assertTrue('BinlogPlayerSourceShardNameMap' in v) - self.assertTrue('0' in v['BinlogPlayerSourceShardNameMap']) + self.assertIn('BinlogPlayerMapSize', v) + self.assertIn('BinlogPlayerSecondsBehindMaster', v) + self.assertIn('BinlogPlayerSecondsBehindMasterMap', v) + self.assertIn('BinlogPlayerSourceShardNameMap', v) + self.assertIn('0', v['BinlogPlayerSourceShardNameMap']) self.assertEquals(v['BinlogPlayerSourceShardNameMap']['0'], 'test_keyspace/80-') - self.assertTrue('BinlogPlayerSourceTabletAliasMap' in v) - self.assertTrue('0' in v['BinlogPlayerSourceTabletAliasMap']) + self.assertIn('BinlogPlayerSourceTabletAliasMap', v) + self.assertIn('0', v['BinlogPlayerSourceTabletAliasMap']) if seconds_behind_master_max != 0: self.assertTrue(v['BinlogPlayerSecondsBehindMaster'] < seconds_behind_master_max, diff --git a/test/update_stream.py b/test/update_stream.py index f8441c8dbf..76363f4ab7 100755 --- a/test/update_stream.py +++ b/test/update_stream.py @@ -210,8 +210,8 @@ class TestUpdateStream(unittest.TestCase): if v['UpdateStreamState'] != 'Enabled': self.fail("Update stream service should be 'Enabled' but is '%s'" % v['UpdateStreamState']) - self.assertTrue('DML' in v['UpdateStreamEvents']) - self.assertTrue('POS' in v['UpdateStreamEvents']) + self.assertIn('DML', v['UpdateStreamEvents']) + self.assertIn('POS', v['UpdateStreamEvents']) logging.debug('Testing enable -> disable switch starting @ %s', start_position) diff --git a/test/vtgatev2_test.py b/test/vtgatev2_test.py index 8377725e04..99e7fe3c00 100755 --- a/test/vtgatev2_test.py +++ b/test/vtgatev2_test.py @@ -291,7 +291,7 @@ class TestVTGateFunctions(unittest.TestCase): # Verify keyspace id for result in cursor.results: kid = result[2] - self.assertTrue(kid in shard_kid_map[shard_names[shard_index]]) + self.assertIn(kid, shard_kid_map[shard_names[shard_index]]) # Do a cross shard range query and assert all rows are fetched cursor = vtgate_conn.cursor(KEYSPACE_NAME, 'master', keyranges=[get_keyrange('75-95')]) From 55d0fe9b9a6a2567bccfb92878a4ec06c99fb499 Mon Sep 17 00:00:00 2001 From: Michael Berlin Date: Mon, 17 Aug 2015 13:26:33 -0700 Subject: [PATCH 23/80] proto: Changed filtered replication lag info from a timestamp to a delay value in query.proto. We're only interested in the fact that the receiving master has seen a lower delay at some point in time. We don't care about the actual timestamps of the filtered replication statements which were applied. --- go/vt/proto/query/query.pb.go | 11 ++++---- proto/query.proto | 11 ++++---- py/vtproto/query_pb2.py | 50 ++++++++++++++++++++++++++++++----- 3 files changed, 54 insertions(+), 18 deletions(-) diff --git a/go/vt/proto/query/query.pb.go b/go/vt/proto/query/query.pb.go index 3be9a634e5..8863538387 100644 --- a/go/vt/proto/query/query.pb.go +++ b/go/vt/proto/query/query.pb.go @@ -802,15 +802,16 @@ type RealtimeStats struct { // how far behind on (MySQL) replication a slave currently is. It is used // by clients for subset selection (so we don't try to send traffic // to tablets that are too far behind). + // NOTE: This field must not be evaluated if "health_error" is not empty. // TODO(mberlin): Let's switch it to int64 instead? SecondsBehindMaster uint32 `protobuf:"varint,2,opt,name=seconds_behind_master" json:"seconds_behind_master,omitempty"` - // filtered_replication_synced_until_timestamp is populated for the receiving + // seconds_behind_master_filtered_replication is populated for the receiving // master of an ongoing filtered replication only. - // It is used to find out how far the receiving master lags behind the - // source shard. - FilteredReplicationSyncedUntilTimestamp int64 `protobuf:"varint,4,opt,name=filtered_replication_synced_until_timestamp" json:"filtered_replication_synced_until_timestamp,omitempty"` + // It specifies how far the receiving master lags behind the sending master. + // NOTE: This field must not be evaluated if "health_error" is not empty. + SecondsBehindMasterFilteredReplication int64 `protobuf:"varint,3,opt,name=seconds_behind_master_filtered_replication" json:"seconds_behind_master_filtered_replication,omitempty"` // cpu_usage is used for load-based balancing - CpuUsage float64 `protobuf:"fixed64,3,opt,name=cpu_usage" json:"cpu_usage,omitempty"` + CpuUsage float64 `protobuf:"fixed64,4,opt,name=cpu_usage" json:"cpu_usage,omitempty"` } func (m *RealtimeStats) Reset() { *m = RealtimeStats{} } diff --git a/proto/query.proto b/proto/query.proto index f393862789..841e15faf5 100644 --- a/proto/query.proto +++ b/proto/query.proto @@ -281,20 +281,19 @@ message RealtimeStats { // we do not send queries to servers that are not healthy. string health_error = 1; - // TODO(mberlin): Add an health error for filtered replication? - // seconds_behind_master is populated for slaves only. It indicates // how far behind on (MySQL) replication a slave currently is. It is used // by clients for subset selection (so we don't try to send traffic // to tablets that are too far behind). + // NOTE: This field must not be evaluated if "health_error" is not empty. // TODO(mberlin): Let's switch it to int64 instead? uint32 seconds_behind_master = 2; - // filtered_replication_synced_until_timestamp is populated for the receiving + // seconds_behind_master_filtered_replication is populated for the receiving // master of an ongoing filtered replication only. - // It is used to find out how far the receiving master lags behind the - // source shard. - int64 filtered_replication_synced_until_timestamp = 3; + // It specifies how far the receiving master lags behind the sending master. + // NOTE: This field must not be evaluated if "health_error" is not empty. + int64 seconds_behind_master_filtered_replication = 3; // cpu_usage is used for load-based balancing double cpu_usage = 4; diff --git a/py/vtproto/query_pb2.py b/py/vtproto/query_pb2.py index 64fdaf517a..b8194d3504 100644 --- a/py/vtproto/query_pb2.py +++ b/py/vtproto/query_pb2.py @@ -20,7 +20,8 @@ import vtrpc_pb2 as vtrpc__pb2 DESCRIPTOR = _descriptor.FileDescriptor( name='query.proto', package='query', - serialized_pb=_b('\n\x0bquery.proto\x12\x05query\x1a\x0etopodata.proto\x1a\x0bvtrpc.proto\"T\n\x06Target\x12\x10\n\x08keyspace\x18\x01 \x01(\t\x12\r\n\x05shard\x18\x02 \x01(\t\x12)\n\x0btablet_type\x18\x03 \x01(\x0e\x32\x14.topodata.TabletType\"\"\n\x0eVTGateCallerID\x12\x10\n\x08username\x18\x01 \x01(\t\"\x92\x03\n\x0c\x42indVariable\x12&\n\x04type\x18\x01 \x01(\x0e\x32\x18.query.BindVariable.Type\x12\x13\n\x0bvalue_bytes\x18\x02 \x01(\x0c\x12\x11\n\tvalue_int\x18\x03 \x01(\x03\x12\x12\n\nvalue_uint\x18\x04 \x01(\x04\x12\x13\n\x0bvalue_float\x18\x05 \x01(\x01\x12\x18\n\x10value_bytes_list\x18\x06 \x03(\x0c\x12\x16\n\x0evalue_int_list\x18\x07 \x03(\x03\x12\x17\n\x0fvalue_uint_list\x18\x08 \x03(\x04\x12\x18\n\x10value_float_list\x18\t \x03(\x01\"\xa3\x01\n\x04Type\x12\r\n\tTYPE_NULL\x10\x00\x12\x0e\n\nTYPE_BYTES\x10\x01\x12\x0c\n\x08TYPE_INT\x10\x02\x12\r\n\tTYPE_UINT\x10\x03\x12\x0e\n\nTYPE_FLOAT\x10\x04\x12\x13\n\x0fTYPE_BYTES_LIST\x10\x05\x12\x11\n\rTYPE_INT_LIST\x10\x06\x12\x12\n\x0eTYPE_UINT_LIST\x10\x07\x12\x13\n\x0fTYPE_FLOAT_LIST\x10\x08\"\xa2\x01\n\nBoundQuery\x12\x0b\n\x03sql\x18\x01 \x01(\x0c\x12<\n\x0e\x62ind_variables\x18\x02 \x03(\x0b\x32$.query.BoundQuery.BindVariablesEntry\x1aI\n\x12\x42indVariablesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\"\n\x05value\x18\x02 \x01(\x0b\x32\x13.query.BindVariable:\x02\x38\x01\"\xa1\x07\n\x05\x46ield\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x1f\n\x04type\x18\x02 \x01(\x0e\x32\x11.query.Field.Type\x12\r\n\x05\x66lags\x18\x03 \x01(\x03\"\xe1\x03\n\x04Type\x12\x10\n\x0cTYPE_DECIMAL\x10\x00\x12\r\n\tTYPE_TINY\x10\x01\x12\x0e\n\nTYPE_SHORT\x10\x02\x12\r\n\tTYPE_LONG\x10\x03\x12\x0e\n\nTYPE_FLOAT\x10\x04\x12\x0f\n\x0bTYPE_DOUBLE\x10\x05\x12\r\n\tTYPE_NULL\x10\x06\x12\x12\n\x0eTYPE_TIMESTAMP\x10\x07\x12\x11\n\rTYPE_LONGLONG\x10\x08\x12\x0e\n\nTYPE_INT24\x10\t\x12\r\n\tTYPE_DATE\x10\n\x12\r\n\tTYPE_TIME\x10\x0b\x12\x11\n\rTYPE_DATETIME\x10\x0c\x12\r\n\tTYPE_YEAR\x10\r\x12\x10\n\x0cTYPE_NEWDATE\x10\x0e\x12\x10\n\x0cTYPE_VARCHAR\x10\x0f\x12\x0c\n\x08TYPE_BIT\x10\x10\x12\x14\n\x0fTYPE_NEWDECIMAL\x10\xf6\x01\x12\x0e\n\tTYPE_ENUM\x10\xf7\x01\x12\r\n\x08TYPE_SET\x10\xf8\x01\x12\x13\n\x0eTYPE_TINY_BLOB\x10\xf9\x01\x12\x15\n\x10TYPE_MEDIUM_BLOB\x10\xfa\x01\x12\x13\n\x0eTYPE_LONG_BLOB\x10\xfb\x01\x12\x0e\n\tTYPE_BLOB\x10\xfc\x01\x12\x14\n\x0fTYPE_VAR_STRING\x10\xfd\x01\x12\x10\n\x0bTYPE_STRING\x10\xfe\x01\x12\x12\n\rTYPE_GEOMETRY\x10\xff\x01\"\xf5\x02\n\x04\x46lag\x12\x15\n\x11VT_ZEROVALUE_FLAG\x10\x00\x12\x14\n\x10VT_NOT_NULL_FLAG\x10\x01\x12\x13\n\x0fVT_PRI_KEY_FLAG\x10\x02\x12\x16\n\x12VT_UNIQUE_KEY_FLAG\x10\x04\x12\x18\n\x14VT_MULTIPLE_KEY_FLAG\x10\x08\x12\x10\n\x0cVT_BLOB_FLAG\x10\x10\x12\x14\n\x10VT_UNSIGNED_FLAG\x10 \x12\x14\n\x10VT_ZEROFILL_FLAG\x10@\x12\x13\n\x0eVT_BINARY_FLAG\x10\x80\x01\x12\x11\n\x0cVT_ENUM_FLAG\x10\x80\x02\x12\x1b\n\x16VT_AUTO_INCREMENT_FLAG\x10\x80\x04\x12\x16\n\x11VT_TIMESTAMP_FLAG\x10\x80\x08\x12\x10\n\x0bVT_SET_FLAG\x10\x80\x10\x12\x1d\n\x18VT_NO_DEFAULT_VALUE_FLAG\x10\x80 \x12\x1a\n\x15VT_ON_UPDATE_NOW_FLAG\x10\x80@\x12\x11\n\x0bVT_NUM_FLAG\x10\x80\x80\x02\"\x15\n\x03Row\x12\x0e\n\x06values\x18\x01 \x03(\x0c\"o\n\x0bQueryResult\x12\x1c\n\x06\x66ields\x18\x01 \x03(\x0b\x32\x0c.query.Field\x12\x15\n\rrows_affected\x18\x02 \x01(\x04\x12\x11\n\tinsert_id\x18\x03 \x01(\x04\x12\x18\n\x04rows\x18\x04 \x03(\x0b\x32\n.query.Row\"\x98\x01\n\x13GetSessionIdRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x10\n\x08keyspace\x18\x03 \x01(\t\x12\r\n\x05shard\x18\x04 \x01(\t\"*\n\x14GetSessionIdResponse\x12\x12\n\nsession_id\x18\x01 \x01(\x03\"\xdf\x01\n\x0e\x45xecuteRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12 \n\x05query\x18\x04 \x01(\x0b\x32\x11.query.BoundQuery\x12\x16\n\x0etransaction_id\x18\x05 \x01(\x03\x12\x12\n\nsession_id\x18\x06 \x01(\x03\"5\n\x0f\x45xecuteResponse\x12\"\n\x06result\x18\x01 \x01(\x0b\x32\x12.query.QueryResult\"\xfe\x01\n\x13\x45xecuteBatchRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12\"\n\x07queries\x18\x04 \x03(\x0b\x32\x11.query.BoundQuery\x12\x16\n\x0e\x61s_transaction\x18\x05 \x01(\x08\x12\x16\n\x0etransaction_id\x18\x06 \x01(\x03\x12\x12\n\nsession_id\x18\x07 \x01(\x03\";\n\x14\x45xecuteBatchResponse\x12#\n\x07results\x18\x01 \x03(\x0b\x32\x12.query.QueryResult\"\xcd\x01\n\x14StreamExecuteRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12 \n\x05query\x18\x04 \x01(\x0b\x32\x11.query.BoundQuery\x12\x12\n\nsession_id\x18\x05 \x01(\x03\";\n\x15StreamExecuteResponse\x12\"\n\x06result\x18\x01 \x01(\x0b\x32\x12.query.QueryResult\"\xa3\x01\n\x0c\x42\x65ginRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12\x12\n\nsession_id\x18\x04 \x01(\x03\"\'\n\rBeginResponse\x12\x16\n\x0etransaction_id\x18\x01 \x01(\x03\"\xbc\x01\n\rCommitRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12\x16\n\x0etransaction_id\x18\x04 \x01(\x03\x12\x12\n\nsession_id\x18\x05 \x01(\x03\"\x10\n\x0e\x43ommitResponse\"\xbe\x01\n\x0fRollbackRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12\x16\n\x0etransaction_id\x18\x04 \x01(\x03\x12\x12\n\nsession_id\x18\x05 \x01(\x03\"\x12\n\x10RollbackResponse\"\xf5\x01\n\x11SplitQueryRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12 \n\x05query\x18\x04 \x01(\x0b\x32\x11.query.BoundQuery\x12\x14\n\x0csplit_column\x18\x05 \x01(\t\x12\x13\n\x0bsplit_count\x18\x06 \x01(\x03\x12\x12\n\nsession_id\x18\x07 \x01(\x03\"A\n\nQuerySplit\x12 \n\x05query\x18\x01 \x01(\x0b\x32\x11.query.BoundQuery\x12\x11\n\trow_count\x18\x02 \x01(\x03\"8\n\x12SplitQueryResponse\x12\"\n\x07queries\x18\x01 \x03(\x0b\x32\x11.query.QuerySplit\"\x15\n\x13StreamHealthRequest\"W\n\rRealtimeStats\x12\x14\n\x0chealth_error\x18\x01 \x01(\t\x12\x1d\n\x15seconds_behind_master\x18\x02 \x01(\r\x12\x11\n\tcpu_usage\x18\x03 \x01(\x01\"\x93\x01\n\x14StreamHealthResponse\x12\x1d\n\x06target\x18\x01 \x01(\x0b\x32\r.query.Target\x12.\n&tablet_externally_reparented_timestamp\x18\x02 \x01(\x03\x12,\n\x0erealtime_stats\x18\x03 \x01(\x0b\x32\x14.query.RealtimeStatsB\x1a\n\x18\x63om.youtube.vitess.protob\x06proto3') + syntax='proto3', + serialized_pb=_b('\n\x0bquery.proto\x12\x05query\x1a\x0etopodata.proto\x1a\x0bvtrpc.proto\"T\n\x06Target\x12\x10\n\x08keyspace\x18\x01 \x01(\t\x12\r\n\x05shard\x18\x02 \x01(\t\x12)\n\x0btablet_type\x18\x03 \x01(\x0e\x32\x14.topodata.TabletType\"\"\n\x0eVTGateCallerID\x12\x10\n\x08username\x18\x01 \x01(\t\"\x92\x03\n\x0c\x42indVariable\x12&\n\x04type\x18\x01 \x01(\x0e\x32\x18.query.BindVariable.Type\x12\x13\n\x0bvalue_bytes\x18\x02 \x01(\x0c\x12\x11\n\tvalue_int\x18\x03 \x01(\x03\x12\x12\n\nvalue_uint\x18\x04 \x01(\x04\x12\x13\n\x0bvalue_float\x18\x05 \x01(\x01\x12\x18\n\x10value_bytes_list\x18\x06 \x03(\x0c\x12\x16\n\x0evalue_int_list\x18\x07 \x03(\x03\x12\x17\n\x0fvalue_uint_list\x18\x08 \x03(\x04\x12\x18\n\x10value_float_list\x18\t \x03(\x01\"\xa3\x01\n\x04Type\x12\r\n\tTYPE_NULL\x10\x00\x12\x0e\n\nTYPE_BYTES\x10\x01\x12\x0c\n\x08TYPE_INT\x10\x02\x12\r\n\tTYPE_UINT\x10\x03\x12\x0e\n\nTYPE_FLOAT\x10\x04\x12\x13\n\x0fTYPE_BYTES_LIST\x10\x05\x12\x11\n\rTYPE_INT_LIST\x10\x06\x12\x12\n\x0eTYPE_UINT_LIST\x10\x07\x12\x13\n\x0fTYPE_FLOAT_LIST\x10\x08\"\xa2\x01\n\nBoundQuery\x12\x0b\n\x03sql\x18\x01 \x01(\x0c\x12<\n\x0e\x62ind_variables\x18\x02 \x03(\x0b\x32$.query.BoundQuery.BindVariablesEntry\x1aI\n\x12\x42indVariablesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\"\n\x05value\x18\x02 \x01(\x0b\x32\x13.query.BindVariable:\x02\x38\x01\"\xa1\x07\n\x05\x46ield\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x1f\n\x04type\x18\x02 \x01(\x0e\x32\x11.query.Field.Type\x12\r\n\x05\x66lags\x18\x03 \x01(\x03\"\xe1\x03\n\x04Type\x12\x10\n\x0cTYPE_DECIMAL\x10\x00\x12\r\n\tTYPE_TINY\x10\x01\x12\x0e\n\nTYPE_SHORT\x10\x02\x12\r\n\tTYPE_LONG\x10\x03\x12\x0e\n\nTYPE_FLOAT\x10\x04\x12\x0f\n\x0bTYPE_DOUBLE\x10\x05\x12\r\n\tTYPE_NULL\x10\x06\x12\x12\n\x0eTYPE_TIMESTAMP\x10\x07\x12\x11\n\rTYPE_LONGLONG\x10\x08\x12\x0e\n\nTYPE_INT24\x10\t\x12\r\n\tTYPE_DATE\x10\n\x12\r\n\tTYPE_TIME\x10\x0b\x12\x11\n\rTYPE_DATETIME\x10\x0c\x12\r\n\tTYPE_YEAR\x10\r\x12\x10\n\x0cTYPE_NEWDATE\x10\x0e\x12\x10\n\x0cTYPE_VARCHAR\x10\x0f\x12\x0c\n\x08TYPE_BIT\x10\x10\x12\x14\n\x0fTYPE_NEWDECIMAL\x10\xf6\x01\x12\x0e\n\tTYPE_ENUM\x10\xf7\x01\x12\r\n\x08TYPE_SET\x10\xf8\x01\x12\x13\n\x0eTYPE_TINY_BLOB\x10\xf9\x01\x12\x15\n\x10TYPE_MEDIUM_BLOB\x10\xfa\x01\x12\x13\n\x0eTYPE_LONG_BLOB\x10\xfb\x01\x12\x0e\n\tTYPE_BLOB\x10\xfc\x01\x12\x14\n\x0fTYPE_VAR_STRING\x10\xfd\x01\x12\x10\n\x0bTYPE_STRING\x10\xfe\x01\x12\x12\n\rTYPE_GEOMETRY\x10\xff\x01\"\xf5\x02\n\x04\x46lag\x12\x15\n\x11VT_ZEROVALUE_FLAG\x10\x00\x12\x14\n\x10VT_NOT_NULL_FLAG\x10\x01\x12\x13\n\x0fVT_PRI_KEY_FLAG\x10\x02\x12\x16\n\x12VT_UNIQUE_KEY_FLAG\x10\x04\x12\x18\n\x14VT_MULTIPLE_KEY_FLAG\x10\x08\x12\x10\n\x0cVT_BLOB_FLAG\x10\x10\x12\x14\n\x10VT_UNSIGNED_FLAG\x10 \x12\x14\n\x10VT_ZEROFILL_FLAG\x10@\x12\x13\n\x0eVT_BINARY_FLAG\x10\x80\x01\x12\x11\n\x0cVT_ENUM_FLAG\x10\x80\x02\x12\x1b\n\x16VT_AUTO_INCREMENT_FLAG\x10\x80\x04\x12\x16\n\x11VT_TIMESTAMP_FLAG\x10\x80\x08\x12\x10\n\x0bVT_SET_FLAG\x10\x80\x10\x12\x1d\n\x18VT_NO_DEFAULT_VALUE_FLAG\x10\x80 \x12\x1a\n\x15VT_ON_UPDATE_NOW_FLAG\x10\x80@\x12\x11\n\x0bVT_NUM_FLAG\x10\x80\x80\x02\"\x15\n\x03Row\x12\x0e\n\x06values\x18\x01 \x03(\x0c\"o\n\x0bQueryResult\x12\x1c\n\x06\x66ields\x18\x01 \x03(\x0b\x32\x0c.query.Field\x12\x15\n\rrows_affected\x18\x02 \x01(\x04\x12\x11\n\tinsert_id\x18\x03 \x01(\x04\x12\x18\n\x04rows\x18\x04 \x03(\x0b\x32\n.query.Row\"\x98\x01\n\x13GetSessionIdRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x10\n\x08keyspace\x18\x03 \x01(\t\x12\r\n\x05shard\x18\x04 \x01(\t\"*\n\x14GetSessionIdResponse\x12\x12\n\nsession_id\x18\x01 \x01(\x03\"\xdf\x01\n\x0e\x45xecuteRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12 \n\x05query\x18\x04 \x01(\x0b\x32\x11.query.BoundQuery\x12\x16\n\x0etransaction_id\x18\x05 \x01(\x03\x12\x12\n\nsession_id\x18\x06 \x01(\x03\"5\n\x0f\x45xecuteResponse\x12\"\n\x06result\x18\x01 \x01(\x0b\x32\x12.query.QueryResult\"\xfe\x01\n\x13\x45xecuteBatchRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12\"\n\x07queries\x18\x04 \x03(\x0b\x32\x11.query.BoundQuery\x12\x16\n\x0e\x61s_transaction\x18\x05 \x01(\x08\x12\x16\n\x0etransaction_id\x18\x06 \x01(\x03\x12\x12\n\nsession_id\x18\x07 \x01(\x03\";\n\x14\x45xecuteBatchResponse\x12#\n\x07results\x18\x01 \x03(\x0b\x32\x12.query.QueryResult\"\xcd\x01\n\x14StreamExecuteRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12 \n\x05query\x18\x04 \x01(\x0b\x32\x11.query.BoundQuery\x12\x12\n\nsession_id\x18\x05 \x01(\x03\";\n\x15StreamExecuteResponse\x12\"\n\x06result\x18\x01 \x01(\x0b\x32\x12.query.QueryResult\"\xa3\x01\n\x0c\x42\x65ginRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12\x12\n\nsession_id\x18\x04 \x01(\x03\"\'\n\rBeginResponse\x12\x16\n\x0etransaction_id\x18\x01 \x01(\x03\"\xbc\x01\n\rCommitRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12\x16\n\x0etransaction_id\x18\x04 \x01(\x03\x12\x12\n\nsession_id\x18\x05 \x01(\x03\"\x10\n\x0e\x43ommitResponse\"\xbe\x01\n\x0fRollbackRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12\x16\n\x0etransaction_id\x18\x04 \x01(\x03\x12\x12\n\nsession_id\x18\x05 \x01(\x03\"\x12\n\x10RollbackResponse\"\xf5\x01\n\x11SplitQueryRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12 \n\x05query\x18\x04 \x01(\x0b\x32\x11.query.BoundQuery\x12\x14\n\x0csplit_column\x18\x05 \x01(\t\x12\x13\n\x0bsplit_count\x18\x06 \x01(\x03\x12\x12\n\nsession_id\x18\x07 \x01(\x03\"A\n\nQuerySplit\x12 \n\x05query\x18\x01 \x01(\x0b\x32\x11.query.BoundQuery\x12\x11\n\trow_count\x18\x02 \x01(\x03\"8\n\x12SplitQueryResponse\x12\"\n\x07queries\x18\x01 \x03(\x0b\x32\x11.query.QuerySplit\"\x15\n\x13StreamHealthRequest\"\x8b\x01\n\rRealtimeStats\x12\x14\n\x0chealth_error\x18\x01 \x01(\t\x12\x1d\n\x15seconds_behind_master\x18\x02 \x01(\r\x12\x32\n*seconds_behind_master_filtered_replication\x18\x03 \x01(\x03\x12\x11\n\tcpu_usage\x18\x04 \x01(\x01\"\x93\x01\n\x14StreamHealthResponse\x12\x1d\n\x06target\x18\x01 \x01(\x0b\x32\r.query.Target\x12.\n&tablet_externally_reparented_timestamp\x18\x02 \x01(\x03\x12,\n\x0erealtime_stats\x18\x03 \x01(\x0b\x32\x14.query.RealtimeStatsB\x1a\n\x18\x63om.youtube.vitess.protob\x06proto3') , dependencies=[topodata__pb2.DESCRIPTOR,vtrpc__pb2.DESCRIPTOR,]) _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -314,6 +315,7 @@ _TARGET = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -344,6 +346,7 @@ _VTGATECALLERID = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -431,6 +434,7 @@ _BINDVARIABLE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -468,6 +472,7 @@ _BOUNDQUERY_BINDVARIABLESENTRY = _descriptor.Descriptor( ], options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')), is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -504,6 +509,7 @@ _BOUNDQUERY = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -550,6 +556,7 @@ _FIELD = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -580,6 +587,7 @@ _ROW = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -631,6 +639,7 @@ _QUERYRESULT = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -682,6 +691,7 @@ _GETSESSIONIDREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -712,6 +722,7 @@ _GETSESSIONIDRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -777,6 +788,7 @@ _EXECUTEREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -807,6 +819,7 @@ _EXECUTERESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -879,6 +892,7 @@ _EXECUTEBATCHREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -909,6 +923,7 @@ _EXECUTEBATCHRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -967,6 +982,7 @@ _STREAMEXECUTEREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -997,6 +1013,7 @@ _STREAMEXECUTERESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1048,6 +1065,7 @@ _BEGINREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1078,6 +1096,7 @@ _BEGINRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1136,6 +1155,7 @@ _COMMITREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1159,6 +1179,7 @@ _COMMITRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1217,6 +1238,7 @@ _ROLLBACKREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1240,6 +1262,7 @@ _ROLLBACKRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1312,6 +1335,7 @@ _SPLITQUERYREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1349,6 +1373,7 @@ _QUERYSPLIT = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1379,6 +1404,7 @@ _SPLITQUERYRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1402,6 +1428,7 @@ _STREAMHEALTHREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1432,8 +1459,15 @@ _REALTIMESTATS = _descriptor.Descriptor( is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='cpu_usage', full_name='query.RealtimeStats.cpu_usage', index=2, - number=3, type=1, cpp_type=5, label=1, + name='seconds_behind_master_filtered_replication', full_name='query.RealtimeStats.seconds_behind_master_filtered_replication', index=2, + number=3, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='cpu_usage', full_name='query.RealtimeStats.cpu_usage', index=3, + number=4, type=1, cpp_type=5, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, @@ -1446,11 +1480,12 @@ _REALTIMESTATS = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], - serialized_start=3903, - serialized_end=3990, + serialized_start=3904, + serialized_end=4043, ) @@ -1490,11 +1525,12 @@ _STREAMHEALTHRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], - serialized_start=3993, - serialized_end=4140, + serialized_start=4046, + serialized_end=4193, ) _TARGET.fields_by_name['tablet_type'].enum_type = topodata__pb2._TABLETTYPE From 3736372feeeeec6daf8b74ba3c2f59143fc83833 Mon Sep 17 00:00:00 2001 From: Shengzhe Yao Date: Mon, 17 Aug 2015 15:33:16 -0700 Subject: [PATCH 24/80] fix splitquery related tests. 1. bind variables in SplitQuery should not contain leading ':'. 2. fix custom_sharding.py test to pass through returned bind variables from VTGate. 3. fix SplitQuery tests in Java. --- go/vt/tabletserver/query_splitter.go | 8 +++--- go/vt/tabletserver/query_splitter_test.go | 26 +++++++++---------- go/vt/tabletserver/sqlquery.go | 6 +++++ .../vitess/vtgate/integration/VtGateIT.java | 25 ++++++++++-------- test/custom_sharding.py | 3 ++- 5 files changed, 39 insertions(+), 29 deletions(-) diff --git a/go/vt/tabletserver/query_splitter.go b/go/vt/tabletserver/query_splitter.go index eac4018b68..664a4f1d79 100644 --- a/go/vt/tabletserver/query_splitter.go +++ b/go/vt/tabletserver/query_splitter.go @@ -28,8 +28,8 @@ type QuerySplitter struct { } const ( - startBindVarName = ":_splitquery_start" - endBindVarName = ":_splitquery_end" + startBindVarName = "_splitquery_start" + endBindVarName = "_splitquery_end" ) // NewQuerySplitter creates a new QuerySplitter. query is the original query @@ -157,7 +157,7 @@ func (qs *QuerySplitter) getWhereClause(whereClause *sqlparser.Where, bindVars m startClause = &sqlparser.ComparisonExpr{ Operator: sqlparser.AST_GE, Left: pk, - Right: sqlparser.ValArg([]byte(startBindVarName)), + Right: sqlparser.ValArg([]byte(":" + startBindVarName)), } if start.IsNumeric() { v, _ := start.ParseInt64() @@ -174,7 +174,7 @@ func (qs *QuerySplitter) getWhereClause(whereClause *sqlparser.Where, bindVars m endClause = &sqlparser.ComparisonExpr{ Operator: sqlparser.AST_LT, Left: pk, - Right: sqlparser.ValArg([]byte(endBindVarName)), + Right: sqlparser.ValArg([]byte(":" + endBindVarName)), } if end.IsNumeric() { v, _ := end.ParseInt64() diff --git a/go/vt/tabletserver/query_splitter_test.go b/go/vt/tabletserver/query_splitter_test.go index 5cf1a66849..353150e5f3 100644 --- a/go/vt/tabletserver/query_splitter_test.go +++ b/go/vt/tabletserver/query_splitter_test.go @@ -165,7 +165,7 @@ func TestGetWhereClause(t *testing.T) { bindVars = make(map[string]interface{}) bindVars[":count"] = 300 clause = splitter.getWhereClause(splitter.sel.Where, bindVars, start, nilValue) - want = " where (count > :count) and (id >= " + startBindVarName + ")" + want = " where (count > :count) and (id >= :" + startBindVarName + ")" got = sqlparser.String(clause) if !reflect.DeepEqual(got, want) { t.Errorf("incorrect where clause, got:%v, want:%v", got, want) @@ -182,7 +182,7 @@ func TestGetWhereClause(t *testing.T) { end, _ := sqltypes.BuildValue(endVal) bindVars = make(map[string]interface{}) clause = splitter.getWhereClause(splitter.sel.Where, bindVars, nilValue, end) - want = " where (count > :count) and (id < " + endBindVarName + ")" + want = " where (count > :count) and (id < :" + endBindVarName + ")" got = sqlparser.String(clause) if !reflect.DeepEqual(got, want) { t.Errorf("incorrect where clause, got:%v, want:%v", got, want) @@ -198,7 +198,7 @@ func TestGetWhereClause(t *testing.T) { // Set both bounds, should add two conditions to where clause bindVars = make(map[string]interface{}) clause = splitter.getWhereClause(splitter.sel.Where, bindVars, start, end) - want = fmt.Sprintf(" where (count > :count) and (id >= %s and id < %s)", startBindVarName, endBindVarName) + want = fmt.Sprintf(" where (count > :count) and (id >= :%s and id < :%s)", startBindVarName, endBindVarName) got = sqlparser.String(clause) if !reflect.DeepEqual(got, want) { t.Errorf("incorrect where clause, got:%v, want:%v", got, want) @@ -219,7 +219,7 @@ func TestGetWhereClause(t *testing.T) { bindVars = make(map[string]interface{}) // Set both bounds, should add two conditions to where clause clause = splitter.getWhereClause(splitter.sel.Where, bindVars, start, end) - want = fmt.Sprintf(" where id >= %s and id < %s", startBindVarName, endBindVarName) + want = fmt.Sprintf(" where id >= :%s and id < :%s", startBindVarName, endBindVarName) got = sqlparser.String(clause) if !reflect.DeepEqual(got, want) { t.Errorf("incorrect where clause, got:%v, want:%v", got, want) @@ -355,18 +355,18 @@ func TestSplitQuery(t *testing.T) { } want := []proto.BoundQuery{ { - Sql: "select * from test_table where (count > :count) and (id < " + endBindVarName + ")", + Sql: "select * from test_table where (count > :count) and (id < :" + endBindVarName + ")", BindVariables: map[string]interface{}{endBindVarName: int64(100)}, }, { - Sql: fmt.Sprintf("select * from test_table where (count > :count) and (id >= %s and id < %s)", startBindVarName, endBindVarName), + Sql: fmt.Sprintf("select * from test_table where (count > :count) and (id >= :%s and id < :%s)", startBindVarName, endBindVarName), BindVariables: map[string]interface{}{ startBindVarName: int64(100), endBindVarName: int64(200), }, }, { - Sql: "select * from test_table where (count > :count) and (id >= " + startBindVarName + ")", + Sql: "select * from test_table where (count > :count) and (id >= :" + startBindVarName + ")", BindVariables: map[string]interface{}{startBindVarName: int64(200)}, }, } @@ -411,18 +411,18 @@ func TestSplitQueryFractionalColumn(t *testing.T) { } want := []proto.BoundQuery{ { - Sql: "select * from test_table where (count > :count) and (id < " + endBindVarName + ")", + Sql: "select * from test_table where (count > :count) and (id < :" + endBindVarName + ")", BindVariables: map[string]interface{}{endBindVarName: 170.5}, }, { - Sql: fmt.Sprintf("select * from test_table where (count > :count) and (id >= %s and id < %s)", startBindVarName, endBindVarName), + Sql: fmt.Sprintf("select * from test_table where (count > :count) and (id >= :%s and id < :%s)", startBindVarName, endBindVarName), BindVariables: map[string]interface{}{ startBindVarName: 170.5, endBindVarName: 330.5, }, }, { - Sql: "select * from test_table where (count > :count) and (id >= " + startBindVarName + ")", + Sql: "select * from test_table where (count > :count) and (id >= :" + startBindVarName + ")", BindVariables: map[string]interface{}{startBindVarName: 330.5}, }, } @@ -451,18 +451,18 @@ func TestSplitQueryStringColumn(t *testing.T) { } want := []proto.BoundQuery{ { - Sql: "select * from test_table where (count > :count) and (id < " + endBindVarName + ")", + Sql: "select * from test_table where (count > :count) and (id < :" + endBindVarName + ")", BindVariables: map[string]interface{}{endBindVarName: hexToByteUInt64(0x55555555)[4:]}, }, { - Sql: fmt.Sprintf("select * from test_table where (count > :count) and (id >= %s and id < %s)", startBindVarName, endBindVarName), + Sql: fmt.Sprintf("select * from test_table where (count > :count) and (id >= :%s and id < :%s)", startBindVarName, endBindVarName), BindVariables: map[string]interface{}{ startBindVarName: hexToByteUInt64(0x55555555)[4:], endBindVarName: hexToByteUInt64(0xAAAAAAAA)[4:], }, }, { - Sql: "select * from test_table where (count > :count) and (id >= " + startBindVarName + ")", + Sql: "select * from test_table where (count > :count) and (id >= :" + startBindVarName + ")", BindVariables: map[string]interface{}{startBindVarName: hexToByteUInt64(0xAAAAAAAA)[4:]}, }, } diff --git a/go/vt/tabletserver/sqlquery.go b/go/vt/tabletserver/sqlquery.go index 9bf862b6bd..fa0b0c8b21 100644 --- a/go/vt/tabletserver/sqlquery.go +++ b/go/vt/tabletserver/sqlquery.go @@ -679,6 +679,9 @@ func getColumnType(qre *QueryExecutor, columnName, tableName string) (int64, err return mproto.VT_NULL, err } defer conn.Recycle() + // TODO(shengzhe): use AST to represent the query to avoid sql injection. + // current code is safe because QuerySplitter.validateQuery is called before + // calling this. query := fmt.Sprintf("SELECT %v FROM %v LIMIT 0", columnName, tableName) result, err := qre.execSQL(conn, query, true) if err != nil { @@ -696,6 +699,9 @@ func getColumnMinMax(qre *QueryExecutor, columnName, tableName string) (*mproto. return nil, err } defer conn.Recycle() + // TODO(shengzhe): use AST to represent the query to avoid sql injection. + // current code is safe because QuerySplitter.validateQuery is called before + // calling this. minMaxSQL := fmt.Sprintf("SELECT MIN(%v), MAX(%v) FROM %v", columnName, columnName, tableName) return qre.execSQL(conn, minMaxSQL, true) } diff --git a/java/vtgate-client/src/test/java/com/youtube/vitess/vtgate/integration/VtGateIT.java b/java/vtgate-client/src/test/java/com/youtube/vitess/vtgate/integration/VtGateIT.java index a68062aede..ed35d3d4e3 100644 --- a/java/vtgate-client/src/test/java/com/youtube/vitess/vtgate/integration/VtGateIT.java +++ b/java/vtgate-client/src/test/java/com/youtube/vitess/vtgate/integration/VtGateIT.java @@ -1,6 +1,7 @@ package com.youtube.vitess.vtgate.integration; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import com.google.common.primitives.UnsignedLong; import com.youtube.vitess.vtgate.BindVariable; @@ -303,13 +304,16 @@ public class VtGateIT { public void testSplitQueryMultipleSplitsPerShard() throws Exception { int rowCount = 30; Util.insertRows(testEnv, 1, 30); - List expectedSqls = - Lists.newArrayList("select id, keyspace_id from vtgate_test where id < 10", - "select id, keyspace_id from vtgate_test where id < 11", - "select id, keyspace_id from vtgate_test where id >= 10 and id < 19", - "select id, keyspace_id from vtgate_test where id >= 11 and id < 19", - "select id, keyspace_id from vtgate_test where id >= 19", - "select id, keyspace_id from vtgate_test where id >= 19"); + Map> expectedSqls = Maps.newHashMap(); + expectedSqls.put("select id, keyspace_id from vtgate_test where id < :_splitquery_end", + Lists.newArrayList(BindVariable.forInt("_splitquery_end", 10))); + expectedSqls.put( + "select id, keyspace_id from vtgate_test where id >= :_splitquery_start " + + "and id < :_splitquery_end", + Lists.newArrayList(BindVariable.forInt("_splitquery_start", 10), + BindVariable.forInt("_splitquery_end", 19))); + expectedSqls.put("select id, keyspace_id from vtgate_test where id >= :_splitquery_start", + Lists.newArrayList(BindVariable.forInt("_splitquery_start", 19))); Util.waitForTablet("rdonly", rowCount, 3, testEnv); VtGate vtgate = VtGate.connect("localhost:" + testEnv.getPort(), 0, testEnv.getRpcClientFactory()); int splitCount = 6; @@ -322,11 +326,11 @@ public class VtGateIT { Set shardsInSplits = new HashSet<>(); for (Query q : queries.keySet()) { String sql = q.getSql(); - Assert.assertTrue(expectedSqls.contains(sql)); - expectedSqls.remove(sql); + List bindVars = expectedSqls.get(sql); + Assert.assertNotNull(bindVars); Assert.assertEquals("test_keyspace", q.getKeyspace()); Assert.assertEquals("rdonly", q.getTabletType()); - Assert.assertEquals(0, q.getBindVars().size()); + Assert.assertEquals(bindVars.size(), q.getBindVars().size()); Assert.assertEquals(null, q.getKeyspaceIds()); String start = Hex.encodeHexString(q.getKeyRanges().get(0).get("Start")); String end = Hex.encodeHexString(q.getKeyRanges().get(0).get("End")); @@ -335,7 +339,6 @@ public class VtGateIT { // Verify the keyrange queries in splits cover the entire keyspace Assert.assertTrue(shardsInSplits.containsAll(testEnv.getShardKidMap().keySet())); - Assert.assertTrue(expectedSqls.size() == 0); } @Test diff --git a/test/custom_sharding.py b/test/custom_sharding.py index 0f90ae235e..a5b314b883 100755 --- a/test/custom_sharding.py +++ b/test/custom_sharding.py @@ -193,7 +193,8 @@ primary key (id) rows = {} for q in s: qr = utils.vtgate.execute_shard(q['QueryShard']['Sql'], - 'test_keyspace', ",".join(q['QueryShard']['Shards'])) + 'test_keyspace', ",".join(q['QueryShard']['Shards']), + tablet_type='master', bindvars=q['QueryShard']['BindVariables']) for r in qr['Rows']: id = int(r[0]) rows[id] = r[1] From 6d36cd24c4bcb1988fa76ae9116b792fb95dc168 Mon Sep 17 00:00:00 2001 From: Alain Jobart Date: Mon, 17 Aug 2015 15:49:12 -0700 Subject: [PATCH 25/80] update_stream.py now uses only vtctl / vtgate. Adding missing APIs to vtctl/s query.go. --- go/vt/vtctl/query.go | 144 +++++++++++++++++++++++++++++++++++++++++- test/tablet.py | 36 +++++++++++ test/update_stream.py | 35 +++------- 3 files changed, 188 insertions(+), 27 deletions(-) diff --git a/go/vt/vtctl/query.go b/go/vt/vtctl/query.go index 616e1c86ed..164421490e 100644 --- a/go/vt/vtctl/query.go +++ b/go/vt/vtctl/query.go @@ -8,6 +8,7 @@ import ( "encoding/json" "flag" "fmt" + "strconv" "strings" "time" @@ -50,6 +51,21 @@ func init() { commandVtTabletExecute, "[-bind_variables ] [-connect_timeout ] [-transaction_id ] [-tablet_type ] -keyspace -shard ", "Executes the given query on the given tablet."}) + addCommand(queriesGroupName, command{ + "VtTabletBegin", + commandVtTabletBegin, + "[-connect_timeout ] [-tablet_type ] -keyspace -shard ", + "Starts a transaction on the provided server."}) + addCommand(queriesGroupName, command{ + "VtTabletCommit", + commandVtTabletCommit, + "[-connect_timeout ] [-tablet_type ] -keyspace -shard ", + "Commits a transaction on the provided server."}) + addCommand(queriesGroupName, command{ + "VtTabletRollback", + commandVtTabletRollback, + "[-connect_timeout ] [-tablet_type ] -keyspace -shard ", + "Rollbacks a transaction on the provided server."}) addCommand(queriesGroupName, command{ "VtTabletStreamHealth", commandVtTabletStreamHealth, @@ -196,7 +212,7 @@ func commandVtTabletExecute(ctx context.Context, wr *wrangler.Wrangler, subFlags return err } if subFlags.NArg() != 2 { - return fmt.Errorf("the and arguments are required for the VtTabletExecute command") + return fmt.Errorf("the and arguments are required for the VtTabletExecute command") } tt, err := topo.ParseTabletType(*tabletType) if err != nil { @@ -228,6 +244,132 @@ func commandVtTabletExecute(ctx context.Context, wr *wrangler.Wrangler, subFlags return printJSON(wr, qr) } +func commandVtTabletBegin(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { + keyspace := subFlags.String("keyspace", "", "keyspace the tablet belongs to") + shard := subFlags.String("shard", "", "shard the tablet belongs to") + tabletType := subFlags.String("tablet_type", "unknown", "tablet type we expect from the tablet (use unknown to use sessionId)") + connectTimeout := subFlags.Duration("connect_timeout", 30*time.Second, "Connection timeout for vttablet client") + if err := subFlags.Parse(args); err != nil { + return err + } + if subFlags.NArg() != 1 { + return fmt.Errorf("the argument is required for the VtTabletBegin command") + } + tt, err := topo.ParseTabletType(*tabletType) + if err != nil { + return err + } + tabletAlias, err := topo.ParseTabletAliasString(subFlags.Arg(0)) + if err != nil { + return err + } + tabletInfo, err := wr.TopoServer().GetTablet(ctx, tabletAlias) + if err != nil { + return err + } + ep, err := topo.TabletEndPoint(tabletInfo.Tablet) + if err != nil { + return fmt.Errorf("cannot get EndPoint from tablet record: %v", err) + } + + conn, err := tabletconn.GetDialer()(ctx, ep, *keyspace, *shard, tt, *connectTimeout) + if err != nil { + return fmt.Errorf("cannot connect to tablet %v: %v", tabletAlias, err) + } + defer conn.Close() + + transactionID, err := conn.Begin(ctx) + if err != nil { + return fmt.Errorf("Begin failed: %v", err) + } + result := map[string]int64{ + "transaction_id": transactionID, + } + return printJSON(wr, result) +} + +func commandVtTabletCommit(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { + keyspace := subFlags.String("keyspace", "", "keyspace the tablet belongs to") + shard := subFlags.String("shard", "", "shard the tablet belongs to") + tabletType := subFlags.String("tablet_type", "unknown", "tablet type we expect from the tablet (use unknown to use sessionId)") + connectTimeout := subFlags.Duration("connect_timeout", 30*time.Second, "Connection timeout for vttablet client") + if err := subFlags.Parse(args); err != nil { + return err + } + if subFlags.NArg() != 2 { + return fmt.Errorf("the and arguments are required for the VtTabletCommit command") + } + transactionID, err := strconv.ParseInt(subFlags.Arg(1), 10, 64) + if err != nil { + return err + } + tt, err := topo.ParseTabletType(*tabletType) + if err != nil { + return err + } + tabletAlias, err := topo.ParseTabletAliasString(subFlags.Arg(0)) + if err != nil { + return err + } + tabletInfo, err := wr.TopoServer().GetTablet(ctx, tabletAlias) + if err != nil { + return err + } + ep, err := topo.TabletEndPoint(tabletInfo.Tablet) + if err != nil { + return fmt.Errorf("cannot get EndPoint from tablet record: %v", err) + } + + conn, err := tabletconn.GetDialer()(ctx, ep, *keyspace, *shard, tt, *connectTimeout) + if err != nil { + return fmt.Errorf("cannot connect to tablet %v: %v", tabletAlias, err) + } + defer conn.Close() + + return conn.Commit(ctx, transactionID) +} + +func commandVtTabletRollback(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { + keyspace := subFlags.String("keyspace", "", "keyspace the tablet belongs to") + shard := subFlags.String("shard", "", "shard the tablet belongs to") + tabletType := subFlags.String("tablet_type", "unknown", "tablet type we expect from the tablet (use unknown to use sessionId)") + connectTimeout := subFlags.Duration("connect_timeout", 30*time.Second, "Connection timeout for vttablet client") + if err := subFlags.Parse(args); err != nil { + return err + } + if subFlags.NArg() != 2 { + return fmt.Errorf("the and arguments are required for the VtTabletRollback command") + } + transactionID, err := strconv.ParseInt(subFlags.Arg(1), 10, 64) + if err != nil { + return err + } + tt, err := topo.ParseTabletType(*tabletType) + if err != nil { + return err + } + tabletAlias, err := topo.ParseTabletAliasString(subFlags.Arg(0)) + if err != nil { + return err + } + tabletInfo, err := wr.TopoServer().GetTablet(ctx, tabletAlias) + if err != nil { + return err + } + ep, err := topo.TabletEndPoint(tabletInfo.Tablet) + if err != nil { + return fmt.Errorf("cannot get EndPoint from tablet record: %v", err) + } + + conn, err := tabletconn.GetDialer()(ctx, ep, *keyspace, *shard, tt, *connectTimeout) + if err != nil { + return fmt.Errorf("cannot connect to tablet %v: %v", tabletAlias, err) + } + defer conn.Close() + + return conn.Rollback(ctx, transactionID) +} + func commandVtTabletStreamHealth(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { count := subFlags.Int("count", 1, "number of responses to wait for") connectTimeout := subFlags.Duration("connect_timeout", 30*time.Second, "Connection timeout for vttablet client") diff --git a/test/tablet.py b/test/tablet.py index f637bf1bd7..7372bc91e3 100644 --- a/test/tablet.py +++ b/test/tablet.py @@ -713,6 +713,42 @@ class Tablet(object): args.extend([self.tablet_alias, sql]) return utils.run_vtctl_json(args, auto_log=auto_log) + def begin(self, auto_log=True): + """begin uses 'vtctl VtTabletBegin' to start a transaction. + """ + args = [ + 'VtTabletBegin', + '-keyspace', self.keyspace, + '-shard', self.shard, + self.tablet_alias, + ] + result = utils.run_vtctl_json(args, auto_log=auto_log) + return result['transaction_id'] + + def commit(self, transaction_id, auto_log=True): + """commit uses 'vtctl VtTabletCommit' to commit a transaction. + """ + args = [ + 'VtTabletCommit', + '-keyspace', self.keyspace, + '-shard', self.shard, + self.tablet_alias, + str(transaction_id), + ] + return utils.run_vtctl(args, auto_log=auto_log) + + def rollback(self, transaction_id, auto_log=True): + """rollback uses 'vtctl VtTabletRollback' to rollback a transaction. + """ + args = [ + 'VtTabletRollback', + '-keyspace', self.keyspace, + '-shard', self.shard, + self.tablet_alias, + str(transaction_id), + ] + return utils.run_vtctl(args, auto_log=auto_log) + def kill_tablets(tablets): for t in tablets: diff --git a/test/update_stream.py b/test/update_stream.py index f8441c8dbf..e061cdb0d3 100755 --- a/test/update_stream.py +++ b/test/update_stream.py @@ -6,22 +6,15 @@ import warnings warnings.simplefilter('ignore') import logging -import os -import time import traceback import threading import unittest -import MySQLdb - import environment import tablet import utils from vtdb import dbexceptions -from vtdb import topology from vtdb import update_stream -from vtdb import vtclient -from zk import zkocc from mysql_flavor import mysql_flavor master_tablet = tablet.Tablet() @@ -69,6 +62,9 @@ def setUpModule(): replica_tablet.init_mysql()] utils.wait_procs(setup_procs) + # start a vtctld so the vtctl insert commands are just RPCs, not forks + utils.Vtctld().start() + # Start up a master mysql and vttablet logging.debug('Setting up tablets') utils.run_vtctl(['CreateKeyspace', 'test_keyspace']) @@ -142,14 +138,6 @@ class TestUpdateStream(unittest.TestCase): "insert into vt_b (eid, name, foo) values (%d, 'name %s', 'foo %s')" % (x, x, x) for x in xrange(count)] - def setUp(self): - self.vtgate_client = zkocc.ZkOccConnection(utils.vtgate.addr(), - 'test_nj', 30.0) - topology.read_topology(self.vtgate_client) - - def tearDown(self): - self.vtgate_client.close() - def _get_master_stream_conn(self): protocol, endpoint = master_tablet.update_stream_python_endpoint() return update_stream.connect(protocol, endpoint, 30) @@ -240,17 +228,12 @@ class TestUpdateStream(unittest.TestCase): self.fail("Update stream returned error '%s'" % str(e)) logging.debug('Streamed %d transactions before exiting', txn_count) - def _exec_vt_txn(self, query_list=None): - if not query_list: - return - vtdb_conn = vtclient.VtOCCConnection(self.vtgate_client, - 'test_keyspace', '0', 'master', 30) - vtdb_conn.connect() - vtdb_cursor = vtdb_conn.cursor() - vtdb_conn.begin() - for q in query_list: - vtdb_cursor.execute(q, {}) - vtdb_conn.commit() + def _exec_vt_txn(self, query_list): + tid = master_tablet.begin(auto_log=False) + for query in query_list: + master_tablet.execute(query, transaction_id=tid, auto_log=False) + master_tablet.commit(tid, auto_log=False) + return def test_stream_parity(self): """test_stream_parity checks the parity of streams received From b50890c91573c44407cd384660cc878d3b13ed2d Mon Sep 17 00:00:00 2001 From: Alain Jobart Date: Mon, 17 Aug 2015 15:51:33 -0700 Subject: [PATCH 26/80] Removing leftover file. --- test/fake_zkocc_config.json | 26 -------------------------- 1 file changed, 26 deletions(-) delete mode 100644 test/fake_zkocc_config.json diff --git a/test/fake_zkocc_config.json b/test/fake_zkocc_config.json deleted file mode 100644 index d67ce080af..0000000000 --- a/test/fake_zkocc_config.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "/zk/testing/vt/ns/test_keyspace": { - "ShardReferences": [ - { - "KeyRange": { - "Start": "c0", - "End": "d0" - }, - "Name": "c0-d0" - } - ] - }, - "/zk/testing/vt/ns/test_keyspace/0/master": { - "entries": [ - { - "uid": 0, - "host": "127.0.0.1", - "port": 0, - "named_port_map": { - "mysql": 3306, - "vt": 6711 - } - } - ] - } -} From c38f9307c746a7564f4ff2460ad35c7db8b27d81 Mon Sep 17 00:00:00 2001 From: Michael Berlin Date: Mon, 17 Aug 2015 16:35:30 -0700 Subject: [PATCH 27/80] vtctl: Adapt WaitForFilteredReplication to use delay instead of timestamp. --- go/vt/vtctl/vtctl.go | 13 ++++++------- .../wait_for_filtered_replication_test.go | 16 ++++++++-------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/go/vt/vtctl/vtctl.go b/go/vt/vtctl/vtctl.go index f0f5c7ba31..8fd0c97ce6 100644 --- a/go/vt/vtctl/vtctl.go +++ b/go/vt/vtctl/vtctl.go @@ -1336,7 +1336,7 @@ func commandShardReplicationFix(ctx context.Context, wr *wrangler.Wrangler, subF func commandWaitForFilteredReplication(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { // Allowed tolerance up to which the local clock may run ahead. Otherwise, the command will fail. - const allowedAheadTime = time.Second + const allowedAheadTime = 2 * time.Second maxDelay := subFlags.Duration("max_delay", 30*time.Second, "Specifies the maximum delay, in seconds, the filtered replication of the"+ " given destination shard should lag behind the source shard. When"+ @@ -1385,12 +1385,12 @@ func commandWaitForFilteredReplication(ctx context.Context, wr *wrangler.Wrangle for { select { case <-ctx.Done(): - return fmt.Errorf("context was done before filtered replication did caught up. Last seen delay: %v context Error: %v", lastSeenDelay, ctx.Err()) + return fmt.Errorf("context was done before filtered replication did catch up. Last seen delay: %v context Error: %v", lastSeenDelay, ctx.Err()) case shr, ok := <-stream: if !ok { return fmt.Errorf("stream ended early: %v", errFunc()) } - gotTarget := shr.GetTarget() + gotTarget := shr.Target if gotTarget == nil { return fmt.Errorf("stream health record did not include Target: %v", shr) } @@ -1401,11 +1401,10 @@ func commandWaitForFilteredReplication(ctx context.Context, wr *wrangler.Wrangle return fmt.Errorf("tablet: %v should be master, but is not. type: %v", alias, gotTarget.TabletType.String()) } - now := time.Now() - lastSync := time.Unix(shr.GetRealtimeStats().FilteredReplicationSyncedUntilTimestamp, 0 /* nsecs */) - lastSeenDelay := now.Sub(lastSync) + delaySecs := shr.RealtimeStats.SecondsBehindMasterFilteredReplication + lastSeenDelay := time.Duration(delaySecs) * time.Second if lastSeenDelay < -allowedAheadTime { - return fmt.Errorf("cannot reliably wait for the filtered replication to catch up. The local clock (runs behind) or the tablet clock (runs ahead) is not synchronized. seen delay: %v local clock now: %v last sync on tablet (%v): %v", lastSeenDelay, now, alias, lastSync) + return fmt.Errorf("cannot reliably wait for the filtered replication to catch up. The tablet clock (runs ahead) is not synchronized. seen delay: %v on tablet: %v", lastSeenDelay, alias) } if lastSeenDelay <= *maxDelay { wr.Logger().Printf("Filtered replication on tablet: %v has caught up. Last seen delay: %.1f seconds\n", alias, lastSeenDelay.Seconds()) diff --git a/go/vt/wrangler/testlib/wait_for_filtered_replication_test.go b/go/vt/wrangler/testlib/wait_for_filtered_replication_test.go index 8b055d64f9..fe33634bd6 100644 --- a/go/vt/wrangler/testlib/wait_for_filtered_replication_test.go +++ b/go/vt/wrangler/testlib/wait_for_filtered_replication_test.go @@ -59,13 +59,13 @@ func TestWaitForFilteredReplication_unsyncClocks(t *testing.T) { // Replication is lagging behind. oneHourDelay := &pbq.RealtimeStats{ - FilteredReplicationSyncedUntilTimestamp: time.Now().Add(-time.Hour).Unix(), + SecondsBehindMasterFilteredReplication: 3600, } - // Tablet is one hour ahead of the local clock. + // Receiving master's clock is running one hour ahead of the sending master. negativeDelayFunc := func() *pbq.RealtimeStats { return &pbq.RealtimeStats{ - FilteredReplicationSyncedUntilTimestamp: time.Now().Add(time.Hour).Unix(), + SecondsBehindMasterFilteredReplication: -3600, } } @@ -80,13 +80,13 @@ func TestWaitForFilteredReplication_unsyncClocksTolerance(t *testing.T) { // Replication is lagging behind. oneHourDelay := &pbq.RealtimeStats{ - FilteredReplicationSyncedUntilTimestamp: time.Now().Add(-time.Hour).Unix(), + SecondsBehindMasterFilteredReplication: 3600, } - // Tablet is half a second ahead of the local clock. + // Tablet is a second ahead of the local clock. slightNegativeDelayFunc := func() *pbq.RealtimeStats { return &pbq.RealtimeStats{ - FilteredReplicationSyncedUntilTimestamp: time.Now().Add(time.Second / 2).Unix(), + SecondsBehindMasterFilteredReplication: -1, } } @@ -96,13 +96,13 @@ func TestWaitForFilteredReplication_unsyncClocksTolerance(t *testing.T) { func waitForFilteredReplicationDefaultDelay(t *testing.T, target *pbq.Target, expectedErr string) { // Replication is lagging behind. oneHourDelay := &pbq.RealtimeStats{ - FilteredReplicationSyncedUntilTimestamp: time.Now().Add(-time.Hour).Unix(), + SecondsBehindMasterFilteredReplication: 3600, } // Replication caught up. oneSecondDelayFunc := func() *pbq.RealtimeStats { return &pbq.RealtimeStats{ - FilteredReplicationSyncedUntilTimestamp: time.Now().Add(-time.Second).Unix(), + SecondsBehindMasterFilteredReplication: 1, } } From 2029abbdade75ece8a6dc78f9377c478200a1fab Mon Sep 17 00:00:00 2001 From: Michael Berlin Date: Mon, 17 Aug 2015 18:13:44 -0700 Subject: [PATCH 28/80] topo: Fix string output of proto3 tablet types. --- go/vt/topo/tablet.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/topo/tablet.go b/go/vt/topo/tablet.go index 75631987d3..96ba682a7b 100644 --- a/go/vt/topo/tablet.go +++ b/go/vt/topo/tablet.go @@ -235,7 +235,7 @@ func IsTypeInList(tabletType pb.TabletType, types []pb.TabletType) bool { func MakeStringTypeList(types []pb.TabletType) []string { strs := make([]string, len(types)) for i, t := range types { - strs[i] = string(t) + strs[i] = t.String() } sort.Strings(strs) return strs From 66433d9f916dd0eb90bd892cb896de5b0aaeda9e Mon Sep 17 00:00:00 2001 From: Michael Berlin Date: Mon, 17 Aug 2015 18:49:21 -0700 Subject: [PATCH 29/80] proto: Add "binlog_players_count" field to RealTimeStats. --- go/vt/proto/query/query.pb.go | 8 +++++++- proto/query.proto | 9 ++++++++- py/vtproto/query_pb2.py | 19 +++++++++++++------ 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/go/vt/proto/query/query.pb.go b/go/vt/proto/query/query.pb.go index 8863538387..4ba7f4972f 100644 --- a/go/vt/proto/query/query.pb.go +++ b/go/vt/proto/query/query.pb.go @@ -809,9 +809,15 @@ type RealtimeStats struct { // master of an ongoing filtered replication only. // It specifies how far the receiving master lags behind the sending master. // NOTE: This field must not be evaluated if "health_error" is not empty. + // NOTE: This field must not be evaluated if "bin_log_players_count" is 0. SecondsBehindMasterFilteredReplication int64 `protobuf:"varint,3,opt,name=seconds_behind_master_filtered_replication" json:"seconds_behind_master_filtered_replication,omitempty"` + // bin_log_players_count is the number of currently running binlog players. + // if the value is 0, it means that filtered replication is currently not + // running on the tablet. If >0, filtered replication is running. + // NOTE: This field must not be evaluated if "health_error" is not empty. + BinlogPlayersCount int32 `protobuf:"varint,4,opt,name=binlog_players_count" json:"binlog_players_count,omitempty"` // cpu_usage is used for load-based balancing - CpuUsage float64 `protobuf:"fixed64,4,opt,name=cpu_usage" json:"cpu_usage,omitempty"` + CpuUsage float64 `protobuf:"fixed64,5,opt,name=cpu_usage" json:"cpu_usage,omitempty"` } func (m *RealtimeStats) Reset() { *m = RealtimeStats{} } diff --git a/proto/query.proto b/proto/query.proto index 841e15faf5..ad88adaba0 100644 --- a/proto/query.proto +++ b/proto/query.proto @@ -293,10 +293,17 @@ message RealtimeStats { // master of an ongoing filtered replication only. // It specifies how far the receiving master lags behind the sending master. // NOTE: This field must not be evaluated if "health_error" is not empty. + // NOTE: This field must not be evaluated if "bin_log_players_count" is 0. int64 seconds_behind_master_filtered_replication = 3; + // bin_log_players_count is the number of currently running binlog players. + // if the value is 0, it means that filtered replication is currently not + // running on the tablet. If >0, filtered replication is running. + // NOTE: This field must not be evaluated if "health_error" is not empty. + int32 binlog_players_count = 4; + // cpu_usage is used for load-based balancing - double cpu_usage = 4; + double cpu_usage = 5; } // StreamHealthResponse is streamed by StreamHealth on a regular basis diff --git a/py/vtproto/query_pb2.py b/py/vtproto/query_pb2.py index b8194d3504..a2947063b2 100644 --- a/py/vtproto/query_pb2.py +++ b/py/vtproto/query_pb2.py @@ -21,7 +21,7 @@ DESCRIPTOR = _descriptor.FileDescriptor( name='query.proto', package='query', syntax='proto3', - serialized_pb=_b('\n\x0bquery.proto\x12\x05query\x1a\x0etopodata.proto\x1a\x0bvtrpc.proto\"T\n\x06Target\x12\x10\n\x08keyspace\x18\x01 \x01(\t\x12\r\n\x05shard\x18\x02 \x01(\t\x12)\n\x0btablet_type\x18\x03 \x01(\x0e\x32\x14.topodata.TabletType\"\"\n\x0eVTGateCallerID\x12\x10\n\x08username\x18\x01 \x01(\t\"\x92\x03\n\x0c\x42indVariable\x12&\n\x04type\x18\x01 \x01(\x0e\x32\x18.query.BindVariable.Type\x12\x13\n\x0bvalue_bytes\x18\x02 \x01(\x0c\x12\x11\n\tvalue_int\x18\x03 \x01(\x03\x12\x12\n\nvalue_uint\x18\x04 \x01(\x04\x12\x13\n\x0bvalue_float\x18\x05 \x01(\x01\x12\x18\n\x10value_bytes_list\x18\x06 \x03(\x0c\x12\x16\n\x0evalue_int_list\x18\x07 \x03(\x03\x12\x17\n\x0fvalue_uint_list\x18\x08 \x03(\x04\x12\x18\n\x10value_float_list\x18\t \x03(\x01\"\xa3\x01\n\x04Type\x12\r\n\tTYPE_NULL\x10\x00\x12\x0e\n\nTYPE_BYTES\x10\x01\x12\x0c\n\x08TYPE_INT\x10\x02\x12\r\n\tTYPE_UINT\x10\x03\x12\x0e\n\nTYPE_FLOAT\x10\x04\x12\x13\n\x0fTYPE_BYTES_LIST\x10\x05\x12\x11\n\rTYPE_INT_LIST\x10\x06\x12\x12\n\x0eTYPE_UINT_LIST\x10\x07\x12\x13\n\x0fTYPE_FLOAT_LIST\x10\x08\"\xa2\x01\n\nBoundQuery\x12\x0b\n\x03sql\x18\x01 \x01(\x0c\x12<\n\x0e\x62ind_variables\x18\x02 \x03(\x0b\x32$.query.BoundQuery.BindVariablesEntry\x1aI\n\x12\x42indVariablesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\"\n\x05value\x18\x02 \x01(\x0b\x32\x13.query.BindVariable:\x02\x38\x01\"\xa1\x07\n\x05\x46ield\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x1f\n\x04type\x18\x02 \x01(\x0e\x32\x11.query.Field.Type\x12\r\n\x05\x66lags\x18\x03 \x01(\x03\"\xe1\x03\n\x04Type\x12\x10\n\x0cTYPE_DECIMAL\x10\x00\x12\r\n\tTYPE_TINY\x10\x01\x12\x0e\n\nTYPE_SHORT\x10\x02\x12\r\n\tTYPE_LONG\x10\x03\x12\x0e\n\nTYPE_FLOAT\x10\x04\x12\x0f\n\x0bTYPE_DOUBLE\x10\x05\x12\r\n\tTYPE_NULL\x10\x06\x12\x12\n\x0eTYPE_TIMESTAMP\x10\x07\x12\x11\n\rTYPE_LONGLONG\x10\x08\x12\x0e\n\nTYPE_INT24\x10\t\x12\r\n\tTYPE_DATE\x10\n\x12\r\n\tTYPE_TIME\x10\x0b\x12\x11\n\rTYPE_DATETIME\x10\x0c\x12\r\n\tTYPE_YEAR\x10\r\x12\x10\n\x0cTYPE_NEWDATE\x10\x0e\x12\x10\n\x0cTYPE_VARCHAR\x10\x0f\x12\x0c\n\x08TYPE_BIT\x10\x10\x12\x14\n\x0fTYPE_NEWDECIMAL\x10\xf6\x01\x12\x0e\n\tTYPE_ENUM\x10\xf7\x01\x12\r\n\x08TYPE_SET\x10\xf8\x01\x12\x13\n\x0eTYPE_TINY_BLOB\x10\xf9\x01\x12\x15\n\x10TYPE_MEDIUM_BLOB\x10\xfa\x01\x12\x13\n\x0eTYPE_LONG_BLOB\x10\xfb\x01\x12\x0e\n\tTYPE_BLOB\x10\xfc\x01\x12\x14\n\x0fTYPE_VAR_STRING\x10\xfd\x01\x12\x10\n\x0bTYPE_STRING\x10\xfe\x01\x12\x12\n\rTYPE_GEOMETRY\x10\xff\x01\"\xf5\x02\n\x04\x46lag\x12\x15\n\x11VT_ZEROVALUE_FLAG\x10\x00\x12\x14\n\x10VT_NOT_NULL_FLAG\x10\x01\x12\x13\n\x0fVT_PRI_KEY_FLAG\x10\x02\x12\x16\n\x12VT_UNIQUE_KEY_FLAG\x10\x04\x12\x18\n\x14VT_MULTIPLE_KEY_FLAG\x10\x08\x12\x10\n\x0cVT_BLOB_FLAG\x10\x10\x12\x14\n\x10VT_UNSIGNED_FLAG\x10 \x12\x14\n\x10VT_ZEROFILL_FLAG\x10@\x12\x13\n\x0eVT_BINARY_FLAG\x10\x80\x01\x12\x11\n\x0cVT_ENUM_FLAG\x10\x80\x02\x12\x1b\n\x16VT_AUTO_INCREMENT_FLAG\x10\x80\x04\x12\x16\n\x11VT_TIMESTAMP_FLAG\x10\x80\x08\x12\x10\n\x0bVT_SET_FLAG\x10\x80\x10\x12\x1d\n\x18VT_NO_DEFAULT_VALUE_FLAG\x10\x80 \x12\x1a\n\x15VT_ON_UPDATE_NOW_FLAG\x10\x80@\x12\x11\n\x0bVT_NUM_FLAG\x10\x80\x80\x02\"\x15\n\x03Row\x12\x0e\n\x06values\x18\x01 \x03(\x0c\"o\n\x0bQueryResult\x12\x1c\n\x06\x66ields\x18\x01 \x03(\x0b\x32\x0c.query.Field\x12\x15\n\rrows_affected\x18\x02 \x01(\x04\x12\x11\n\tinsert_id\x18\x03 \x01(\x04\x12\x18\n\x04rows\x18\x04 \x03(\x0b\x32\n.query.Row\"\x98\x01\n\x13GetSessionIdRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x10\n\x08keyspace\x18\x03 \x01(\t\x12\r\n\x05shard\x18\x04 \x01(\t\"*\n\x14GetSessionIdResponse\x12\x12\n\nsession_id\x18\x01 \x01(\x03\"\xdf\x01\n\x0e\x45xecuteRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12 \n\x05query\x18\x04 \x01(\x0b\x32\x11.query.BoundQuery\x12\x16\n\x0etransaction_id\x18\x05 \x01(\x03\x12\x12\n\nsession_id\x18\x06 \x01(\x03\"5\n\x0f\x45xecuteResponse\x12\"\n\x06result\x18\x01 \x01(\x0b\x32\x12.query.QueryResult\"\xfe\x01\n\x13\x45xecuteBatchRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12\"\n\x07queries\x18\x04 \x03(\x0b\x32\x11.query.BoundQuery\x12\x16\n\x0e\x61s_transaction\x18\x05 \x01(\x08\x12\x16\n\x0etransaction_id\x18\x06 \x01(\x03\x12\x12\n\nsession_id\x18\x07 \x01(\x03\";\n\x14\x45xecuteBatchResponse\x12#\n\x07results\x18\x01 \x03(\x0b\x32\x12.query.QueryResult\"\xcd\x01\n\x14StreamExecuteRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12 \n\x05query\x18\x04 \x01(\x0b\x32\x11.query.BoundQuery\x12\x12\n\nsession_id\x18\x05 \x01(\x03\";\n\x15StreamExecuteResponse\x12\"\n\x06result\x18\x01 \x01(\x0b\x32\x12.query.QueryResult\"\xa3\x01\n\x0c\x42\x65ginRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12\x12\n\nsession_id\x18\x04 \x01(\x03\"\'\n\rBeginResponse\x12\x16\n\x0etransaction_id\x18\x01 \x01(\x03\"\xbc\x01\n\rCommitRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12\x16\n\x0etransaction_id\x18\x04 \x01(\x03\x12\x12\n\nsession_id\x18\x05 \x01(\x03\"\x10\n\x0e\x43ommitResponse\"\xbe\x01\n\x0fRollbackRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12\x16\n\x0etransaction_id\x18\x04 \x01(\x03\x12\x12\n\nsession_id\x18\x05 \x01(\x03\"\x12\n\x10RollbackResponse\"\xf5\x01\n\x11SplitQueryRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12 \n\x05query\x18\x04 \x01(\x0b\x32\x11.query.BoundQuery\x12\x14\n\x0csplit_column\x18\x05 \x01(\t\x12\x13\n\x0bsplit_count\x18\x06 \x01(\x03\x12\x12\n\nsession_id\x18\x07 \x01(\x03\"A\n\nQuerySplit\x12 \n\x05query\x18\x01 \x01(\x0b\x32\x11.query.BoundQuery\x12\x11\n\trow_count\x18\x02 \x01(\x03\"8\n\x12SplitQueryResponse\x12\"\n\x07queries\x18\x01 \x03(\x0b\x32\x11.query.QuerySplit\"\x15\n\x13StreamHealthRequest\"\x8b\x01\n\rRealtimeStats\x12\x14\n\x0chealth_error\x18\x01 \x01(\t\x12\x1d\n\x15seconds_behind_master\x18\x02 \x01(\r\x12\x32\n*seconds_behind_master_filtered_replication\x18\x03 \x01(\x03\x12\x11\n\tcpu_usage\x18\x04 \x01(\x01\"\x93\x01\n\x14StreamHealthResponse\x12\x1d\n\x06target\x18\x01 \x01(\x0b\x32\r.query.Target\x12.\n&tablet_externally_reparented_timestamp\x18\x02 \x01(\x03\x12,\n\x0erealtime_stats\x18\x03 \x01(\x0b\x32\x14.query.RealtimeStatsB\x1a\n\x18\x63om.youtube.vitess.protob\x06proto3') + serialized_pb=_b('\n\x0bquery.proto\x12\x05query\x1a\x0etopodata.proto\x1a\x0bvtrpc.proto\"T\n\x06Target\x12\x10\n\x08keyspace\x18\x01 \x01(\t\x12\r\n\x05shard\x18\x02 \x01(\t\x12)\n\x0btablet_type\x18\x03 \x01(\x0e\x32\x14.topodata.TabletType\"\"\n\x0eVTGateCallerID\x12\x10\n\x08username\x18\x01 \x01(\t\"\x92\x03\n\x0c\x42indVariable\x12&\n\x04type\x18\x01 \x01(\x0e\x32\x18.query.BindVariable.Type\x12\x13\n\x0bvalue_bytes\x18\x02 \x01(\x0c\x12\x11\n\tvalue_int\x18\x03 \x01(\x03\x12\x12\n\nvalue_uint\x18\x04 \x01(\x04\x12\x13\n\x0bvalue_float\x18\x05 \x01(\x01\x12\x18\n\x10value_bytes_list\x18\x06 \x03(\x0c\x12\x16\n\x0evalue_int_list\x18\x07 \x03(\x03\x12\x17\n\x0fvalue_uint_list\x18\x08 \x03(\x04\x12\x18\n\x10value_float_list\x18\t \x03(\x01\"\xa3\x01\n\x04Type\x12\r\n\tTYPE_NULL\x10\x00\x12\x0e\n\nTYPE_BYTES\x10\x01\x12\x0c\n\x08TYPE_INT\x10\x02\x12\r\n\tTYPE_UINT\x10\x03\x12\x0e\n\nTYPE_FLOAT\x10\x04\x12\x13\n\x0fTYPE_BYTES_LIST\x10\x05\x12\x11\n\rTYPE_INT_LIST\x10\x06\x12\x12\n\x0eTYPE_UINT_LIST\x10\x07\x12\x13\n\x0fTYPE_FLOAT_LIST\x10\x08\"\xa2\x01\n\nBoundQuery\x12\x0b\n\x03sql\x18\x01 \x01(\x0c\x12<\n\x0e\x62ind_variables\x18\x02 \x03(\x0b\x32$.query.BoundQuery.BindVariablesEntry\x1aI\n\x12\x42indVariablesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\"\n\x05value\x18\x02 \x01(\x0b\x32\x13.query.BindVariable:\x02\x38\x01\"\xa1\x07\n\x05\x46ield\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x1f\n\x04type\x18\x02 \x01(\x0e\x32\x11.query.Field.Type\x12\r\n\x05\x66lags\x18\x03 \x01(\x03\"\xe1\x03\n\x04Type\x12\x10\n\x0cTYPE_DECIMAL\x10\x00\x12\r\n\tTYPE_TINY\x10\x01\x12\x0e\n\nTYPE_SHORT\x10\x02\x12\r\n\tTYPE_LONG\x10\x03\x12\x0e\n\nTYPE_FLOAT\x10\x04\x12\x0f\n\x0bTYPE_DOUBLE\x10\x05\x12\r\n\tTYPE_NULL\x10\x06\x12\x12\n\x0eTYPE_TIMESTAMP\x10\x07\x12\x11\n\rTYPE_LONGLONG\x10\x08\x12\x0e\n\nTYPE_INT24\x10\t\x12\r\n\tTYPE_DATE\x10\n\x12\r\n\tTYPE_TIME\x10\x0b\x12\x11\n\rTYPE_DATETIME\x10\x0c\x12\r\n\tTYPE_YEAR\x10\r\x12\x10\n\x0cTYPE_NEWDATE\x10\x0e\x12\x10\n\x0cTYPE_VARCHAR\x10\x0f\x12\x0c\n\x08TYPE_BIT\x10\x10\x12\x14\n\x0fTYPE_NEWDECIMAL\x10\xf6\x01\x12\x0e\n\tTYPE_ENUM\x10\xf7\x01\x12\r\n\x08TYPE_SET\x10\xf8\x01\x12\x13\n\x0eTYPE_TINY_BLOB\x10\xf9\x01\x12\x15\n\x10TYPE_MEDIUM_BLOB\x10\xfa\x01\x12\x13\n\x0eTYPE_LONG_BLOB\x10\xfb\x01\x12\x0e\n\tTYPE_BLOB\x10\xfc\x01\x12\x14\n\x0fTYPE_VAR_STRING\x10\xfd\x01\x12\x10\n\x0bTYPE_STRING\x10\xfe\x01\x12\x12\n\rTYPE_GEOMETRY\x10\xff\x01\"\xf5\x02\n\x04\x46lag\x12\x15\n\x11VT_ZEROVALUE_FLAG\x10\x00\x12\x14\n\x10VT_NOT_NULL_FLAG\x10\x01\x12\x13\n\x0fVT_PRI_KEY_FLAG\x10\x02\x12\x16\n\x12VT_UNIQUE_KEY_FLAG\x10\x04\x12\x18\n\x14VT_MULTIPLE_KEY_FLAG\x10\x08\x12\x10\n\x0cVT_BLOB_FLAG\x10\x10\x12\x14\n\x10VT_UNSIGNED_FLAG\x10 \x12\x14\n\x10VT_ZEROFILL_FLAG\x10@\x12\x13\n\x0eVT_BINARY_FLAG\x10\x80\x01\x12\x11\n\x0cVT_ENUM_FLAG\x10\x80\x02\x12\x1b\n\x16VT_AUTO_INCREMENT_FLAG\x10\x80\x04\x12\x16\n\x11VT_TIMESTAMP_FLAG\x10\x80\x08\x12\x10\n\x0bVT_SET_FLAG\x10\x80\x10\x12\x1d\n\x18VT_NO_DEFAULT_VALUE_FLAG\x10\x80 \x12\x1a\n\x15VT_ON_UPDATE_NOW_FLAG\x10\x80@\x12\x11\n\x0bVT_NUM_FLAG\x10\x80\x80\x02\"\x15\n\x03Row\x12\x0e\n\x06values\x18\x01 \x03(\x0c\"o\n\x0bQueryResult\x12\x1c\n\x06\x66ields\x18\x01 \x03(\x0b\x32\x0c.query.Field\x12\x15\n\rrows_affected\x18\x02 \x01(\x04\x12\x11\n\tinsert_id\x18\x03 \x01(\x04\x12\x18\n\x04rows\x18\x04 \x03(\x0b\x32\n.query.Row\"\x98\x01\n\x13GetSessionIdRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x10\n\x08keyspace\x18\x03 \x01(\t\x12\r\n\x05shard\x18\x04 \x01(\t\"*\n\x14GetSessionIdResponse\x12\x12\n\nsession_id\x18\x01 \x01(\x03\"\xdf\x01\n\x0e\x45xecuteRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12 \n\x05query\x18\x04 \x01(\x0b\x32\x11.query.BoundQuery\x12\x16\n\x0etransaction_id\x18\x05 \x01(\x03\x12\x12\n\nsession_id\x18\x06 \x01(\x03\"5\n\x0f\x45xecuteResponse\x12\"\n\x06result\x18\x01 \x01(\x0b\x32\x12.query.QueryResult\"\xfe\x01\n\x13\x45xecuteBatchRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12\"\n\x07queries\x18\x04 \x03(\x0b\x32\x11.query.BoundQuery\x12\x16\n\x0e\x61s_transaction\x18\x05 \x01(\x08\x12\x16\n\x0etransaction_id\x18\x06 \x01(\x03\x12\x12\n\nsession_id\x18\x07 \x01(\x03\";\n\x14\x45xecuteBatchResponse\x12#\n\x07results\x18\x01 \x03(\x0b\x32\x12.query.QueryResult\"\xcd\x01\n\x14StreamExecuteRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12 \n\x05query\x18\x04 \x01(\x0b\x32\x11.query.BoundQuery\x12\x12\n\nsession_id\x18\x05 \x01(\x03\";\n\x15StreamExecuteResponse\x12\"\n\x06result\x18\x01 \x01(\x0b\x32\x12.query.QueryResult\"\xa3\x01\n\x0c\x42\x65ginRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12\x12\n\nsession_id\x18\x04 \x01(\x03\"\'\n\rBeginResponse\x12\x16\n\x0etransaction_id\x18\x01 \x01(\x03\"\xbc\x01\n\rCommitRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12\x16\n\x0etransaction_id\x18\x04 \x01(\x03\x12\x12\n\nsession_id\x18\x05 \x01(\x03\"\x10\n\x0e\x43ommitResponse\"\xbe\x01\n\x0fRollbackRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12\x16\n\x0etransaction_id\x18\x04 \x01(\x03\x12\x12\n\nsession_id\x18\x05 \x01(\x03\"\x12\n\x10RollbackResponse\"\xf5\x01\n\x11SplitQueryRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12 \n\x05query\x18\x04 \x01(\x0b\x32\x11.query.BoundQuery\x12\x14\n\x0csplit_column\x18\x05 \x01(\t\x12\x13\n\x0bsplit_count\x18\x06 \x01(\x03\x12\x12\n\nsession_id\x18\x07 \x01(\x03\"A\n\nQuerySplit\x12 \n\x05query\x18\x01 \x01(\x0b\x32\x11.query.BoundQuery\x12\x11\n\trow_count\x18\x02 \x01(\x03\"8\n\x12SplitQueryResponse\x12\"\n\x07queries\x18\x01 \x03(\x0b\x32\x11.query.QuerySplit\"\x15\n\x13StreamHealthRequest\"\xa9\x01\n\rRealtimeStats\x12\x14\n\x0chealth_error\x18\x01 \x01(\t\x12\x1d\n\x15seconds_behind_master\x18\x02 \x01(\r\x12\x32\n*seconds_behind_master_filtered_replication\x18\x03 \x01(\x03\x12\x1c\n\x14\x62inlog_players_count\x18\x04 \x01(\x05\x12\x11\n\tcpu_usage\x18\x05 \x01(\x01\"\x93\x01\n\x14StreamHealthResponse\x12\x1d\n\x06target\x18\x01 \x01(\x0b\x32\r.query.Target\x12.\n&tablet_externally_reparented_timestamp\x18\x02 \x01(\x03\x12,\n\x0erealtime_stats\x18\x03 \x01(\x0b\x32\x14.query.RealtimeStatsB\x1a\n\x18\x63om.youtube.vitess.protob\x06proto3') , dependencies=[topodata__pb2.DESCRIPTOR,vtrpc__pb2.DESCRIPTOR,]) _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -1466,8 +1466,15 @@ _REALTIMESTATS = _descriptor.Descriptor( is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='cpu_usage', full_name='query.RealtimeStats.cpu_usage', index=3, - number=4, type=1, cpp_type=5, label=1, + name='binlog_players_count', full_name='query.RealtimeStats.binlog_players_count', index=3, + number=4, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='cpu_usage', full_name='query.RealtimeStats.cpu_usage', index=4, + number=5, type=1, cpp_type=5, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, @@ -1485,7 +1492,7 @@ _REALTIMESTATS = _descriptor.Descriptor( oneofs=[ ], serialized_start=3904, - serialized_end=4043, + serialized_end=4073, ) @@ -1529,8 +1536,8 @@ _STREAMHEALTHRESPONSE = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=4046, - serialized_end=4193, + serialized_start=4076, + serialized_end=4223, ) _TARGET.fields_by_name['tablet_type'].enum_type = topodata__pb2._TABLETTYPE From f1e25c869beaa469ab244a325af33a6ec890609c Mon Sep 17 00:00:00 2001 From: Michael Berlin Date: Mon, 17 Aug 2015 18:51:16 -0700 Subject: [PATCH 30/80] test: Remove unused imports from tabletmanager.py. --- test/tabletmanager.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/test/tabletmanager.py b/test/tabletmanager.py index 20031965c8..19ae249a9d 100755 --- a/test/tabletmanager.py +++ b/test/tabletmanager.py @@ -7,10 +7,6 @@ warnings.simplefilter("ignore") import json import logging -import os -import signal -from subprocess import PIPE -import threading import time import unittest import urllib @@ -21,7 +17,6 @@ import utils import tablet from mysql_flavor import mysql_flavor from protocols_flavor import protocols_flavor -from vtdb import dbexceptions tablet_62344 = tablet.Tablet(62344) tablet_62044 = tablet.Tablet(62044) From 72679bc1f2a15ad751aa77490cde279fe3d378dd Mon Sep 17 00:00:00 2001 From: Michael Berlin Date: Mon, 17 Aug 2015 18:57:14 -0700 Subject: [PATCH 31/80] tabletmanager: Populate new StreamHealth field "seconds_behind_master_filtered_replication". --- go/vt/tabletmanager/binlog.go | 62 ++++++++++++++++++++++-------- go/vt/tabletmanager/healthcheck.go | 5 ++- 2 files changed, 51 insertions(+), 16 deletions(-) diff --git a/go/vt/tabletmanager/binlog.go b/go/vt/tabletmanager/binlog.go index 083a9f2a1d..cd6df4652b 100644 --- a/go/vt/tabletmanager/binlog.go +++ b/go/vt/tabletmanager/binlog.go @@ -332,18 +332,15 @@ func NewBinlogPlayerMap(ts topo.Server, dbConfig *sqldb.ConnParams, mysqld mysql // RegisterBinlogPlayerMap registers the varz for the players. func RegisterBinlogPlayerMap(blm *BinlogPlayerMap) { - stats.Publish("BinlogPlayerMapSize", stats.IntFunc(blm.size)) - stats.Publish("BinlogPlayerSecondsBehindMaster", stats.IntFunc(func() int64 { - sbm := int64(0) + stats.Publish("BinlogPlayerMapSize", stats.IntFunc(stats.IntFunc(func() int64 { blm.mu.Lock() - for _, bpc := range blm.players { - psbm := bpc.binlogPlayerStats.SecondsBehindMaster.Get() - if psbm > sbm { - sbm = psbm - } - } - blm.mu.Unlock() - return sbm + defer blm.mu.Unlock() + return int64(len(blm.players)) + }))) + stats.Publish("BinlogPlayerSecondsBehindMaster", stats.IntFunc(func() int64 { + blm.mu.Lock() + defer blm.mu.Unlock() + return blm.maxSecondsBehindMaster_UNGUARDED() })) stats.Publish("BinlogPlayerSecondsBehindMasterMap", stats.CountersFunc(func() map[string]int64 { blm.mu.Lock() @@ -378,11 +375,24 @@ func RegisterBinlogPlayerMap(blm *BinlogPlayerMap) { })) } -func (blm *BinlogPlayerMap) size() int64 { +func (blm *BinlogPlayerMap) isRunningFilteredReplication() bool { blm.mu.Lock() - result := len(blm.players) - blm.mu.Unlock() - return int64(result) + defer blm.mu.Unlock() + return len(blm.players) != 0 +} + +// maxSecondsBehindMaster returns the maximum of the secondsBehindMaster +// value of all binlog players i.e. the highest seen filtered replication lag. +// NOTE: Caller must own a lock on blm.mu. +func (blm *BinlogPlayerMap) maxSecondsBehindMaster_UNGUARDED() int64 { + sbm := int64(0) + for _, bpc := range blm.players { + psbm := bpc.binlogPlayerStats.SecondsBehindMaster.Get() + if psbm > sbm { + sbm = psbm + } + } + return sbm } // addPlayer adds a new player to the map. It assumes we have the lock. @@ -624,6 +634,7 @@ type BinlogPlayerMapStatus struct { } // Status returns the BinlogPlayerMapStatus for the BinlogPlayerMap. +// It is used to display the complete status in the webinterface. func (blm *BinlogPlayerMap) Status() *BinlogPlayerMapStatus { // Create the result, take care of the stopped state. result := &BinlogPlayerMapStatus{} @@ -667,3 +678,24 @@ func (blm *BinlogPlayerMap) Status() *BinlogPlayerMapStatus { return result } + +// BinlogPlayerMapStatusSummary contains aggregated health information e.g. +// the maximum replication delay across all binlog players. +type BinlogPlayerMapStatusSummary struct { + // TODO(mberlin): Record here LastError and State != Running as well? + maxSecondsBehindMaster int64 + binlogPlayersCount int32 +} + +// StatusSummary returns a summary of the state of all active binlog players. +// It is used by the QueryService.StreamHealth RPC. +func (blm *BinlogPlayerMap) StatusSummary() *BinlogPlayerMapStatusSummary { + result := &BinlogPlayerMapStatusSummary{} + + blm.mu.Lock() + defer blm.mu.Unlock() + result.maxSecondsBehindMaster = blm.maxSecondsBehindMaster_UNGUARDED() + result.binlogPlayersCount = int32(len(blm.players)) + + return result +} diff --git a/go/vt/tabletmanager/healthcheck.go b/go/vt/tabletmanager/healthcheck.go index cb007daf59..10df3463ad 100644 --- a/go/vt/tabletmanager/healthcheck.go +++ b/go/vt/tabletmanager/healthcheck.go @@ -147,7 +147,7 @@ func (agent *ActionAgent) runHealthCheck(targetTabletType pbt.TabletType) { // figure out if we should be running the query service shouldQueryServiceBeRunning := false var blacklistedTables []string - if topo.IsRunningQueryService(targetTabletType) && agent.BinlogPlayerMap.size() == 0 { + if topo.IsRunningQueryService(targetTabletType) && !agent.BinlogPlayerMap.isRunningFilteredReplication() { shouldQueryServiceBeRunning = true if tabletControl != nil { blacklistedTables = tabletControl.BlacklistedTables @@ -259,6 +259,9 @@ func (agent *ActionAgent) runHealthCheck(targetTabletType pbt.TabletType) { stats := &pb.RealtimeStats{ SecondsBehindMaster: uint32(replicationDelay.Seconds()), } + blpStats := agent.BinlogPlayerMap.StatusSummary() + stats.SecondsBehindMasterFilteredReplication = blpStats.maxSecondsBehindMaster + stats.BinlogPlayersCount = blpStats.binlogPlayersCount if err != nil { stats.HealthError = err.Error() } From 0d7f7a0f1aaea23184a81a975d00e727a28c2bc8 Mon Sep 17 00:00:00 2001 From: Michael Berlin Date: Mon, 17 Aug 2015 18:58:41 -0700 Subject: [PATCH 32/80] test: Verify that QueryService.StreamHealth returns correct stats of the filtered replication. We do so by comparing the BinlogPlayer vars against the output of vtctl VtTabletStreamHealth. --- go/vt/vtctl/vtctl.go | 2 +- test/resharding.py | 27 +++++++++++++++++++++++++-- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/go/vt/vtctl/vtctl.go b/go/vt/vtctl/vtctl.go index 8fd0c97ce6..089594e8c3 100644 --- a/go/vt/vtctl/vtctl.go +++ b/go/vt/vtctl/vtctl.go @@ -911,7 +911,7 @@ func commandRunHealthCheck(ctx context.Context, wr *wrangler.Wrangler, subFlags if err != nil { return err } - servedType, err := parseTabletType(subFlags.Arg(1), []pb.TabletType{pb.TabletType_REPLICA, pb.TabletType_RDONLY}) + servedType, err := parseTabletType(subFlags.Arg(1), []pb.TabletType{pb.TabletType_MASTER, pb.TabletType_REPLICA, pb.TabletType_RDONLY}) if err != nil { return err } diff --git a/test/resharding.py b/test/resharding.py index 764b6f08b6..9a0d63ad53 100755 --- a/test/resharding.py +++ b/test/resharding.py @@ -5,10 +5,10 @@ # be found in the LICENSE file. import base64 +import json import logging -import os -import threading import struct +import threading import time import unittest @@ -404,6 +404,26 @@ primary key (name) v['BinlogPlayerSecondsBehindMasterMap']['0'], seconds_behind_master_max)) + def _check_stream_health_equals_binlog_player_vars(self, tablet): + blp_stats = utils.get_vars(tablet.port) + # Enforce health check because it's not running by default as tablets are not started with it. + utils.run_vtctl(["RunHealthCheck", tablet.tablet_alias, 'master']) + stream_health, _ = utils.run_vtctl(['VtTabletStreamHealth', + '-count', '1', + tablet.tablet_alias], + trap_output=True, auto_log=True) + logging.debug("Got health: %s", stream_health) + data = json.loads(stream_health) + self.assertIn('realtime_stats', data) + self.assertNotIn('health_error', data['realtime_stats']) + # count is > 0 and therefore not omitted by the Go JSON marshaller. + self.assertIn('binlog_players_count', data['realtime_stats']) + self.assertEqual(blp_stats['BinlogPlayerMapSize'], + data['realtime_stats']['binlog_players_count']) + self.assertEqual(blp_stats['BinlogPlayerSecondsBehindMaster'], + data['realtime_stats'].get( + 'seconds_behind_master_filtered_replication', 0)) + def _test_keyrange_constraints(self): with self.assertRaisesRegexp(dbexceptions.DatabaseError, '.*enforce keyspace_id range.*'): self._exec_dml( @@ -553,6 +573,9 @@ primary key (name) # check that binlog server exported the stats vars self._check_binlog_server_vars(shard_1_slave1) + self._check_stream_health_equals_binlog_player_vars(shard_2_master) + self._check_stream_health_equals_binlog_player_vars(shard_3_master) + # testing filtered replication: insert a bunch of data on shard 1, # check we get most of it after a few seconds, wait for binlog server # timeout, check we get all of it. From a207187b507d6b6066a46de19db3a3f15b600801 Mon Sep 17 00:00:00 2001 From: Michael Berlin Date: Mon, 17 Aug 2015 19:03:36 -0700 Subject: [PATCH 33/80] vtctl: WaitForFilteredReplication: Remove special handling for tolerated negative delays. A negative delay should never be sent back by the tablet. It always sends a minimum value of 0. --- go/vt/vtctl/vtctl.go | 6 +-- .../wait_for_filtered_replication_test.go | 41 ------------------- 2 files changed, 2 insertions(+), 45 deletions(-) diff --git a/go/vt/vtctl/vtctl.go b/go/vt/vtctl/vtctl.go index 089594e8c3..7b7cfc0f2d 100644 --- a/go/vt/vtctl/vtctl.go +++ b/go/vt/vtctl/vtctl.go @@ -1335,8 +1335,6 @@ func commandShardReplicationFix(ctx context.Context, wr *wrangler.Wrangler, subF } func commandWaitForFilteredReplication(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { - // Allowed tolerance up to which the local clock may run ahead. Otherwise, the command will fail. - const allowedAheadTime = 2 * time.Second maxDelay := subFlags.Duration("max_delay", 30*time.Second, "Specifies the maximum delay, in seconds, the filtered replication of the"+ " given destination shard should lag behind the source shard. When"+ @@ -1403,8 +1401,8 @@ func commandWaitForFilteredReplication(ctx context.Context, wr *wrangler.Wrangle delaySecs := shr.RealtimeStats.SecondsBehindMasterFilteredReplication lastSeenDelay := time.Duration(delaySecs) * time.Second - if lastSeenDelay < -allowedAheadTime { - return fmt.Errorf("cannot reliably wait for the filtered replication to catch up. The tablet clock (runs ahead) is not synchronized. seen delay: %v on tablet: %v", lastSeenDelay, alias) + if lastSeenDelay < 0 { + return fmt.Errorf("last seen delay should never be negative. tablet: %v delay: %v", alias, lastSeenDelay) } if lastSeenDelay <= *maxDelay { wr.Logger().Printf("Filtered replication on tablet: %v has caught up. Last seen delay: %.1f seconds\n", alias, lastSeenDelay.Seconds()) diff --git a/go/vt/wrangler/testlib/wait_for_filtered_replication_test.go b/go/vt/wrangler/testlib/wait_for_filtered_replication_test.go index fe33634bd6..abec8a81d9 100644 --- a/go/vt/wrangler/testlib/wait_for_filtered_replication_test.go +++ b/go/vt/wrangler/testlib/wait_for_filtered_replication_test.go @@ -52,47 +52,6 @@ func TestWaitForFilteredReplication_wrongTarget(t *testing.T) { waitForFilteredReplicationDefaultDelay(t, target, "received health record for wrong tablet") } -// TestWaitForFilteredReplication_unsyncClocks tests that -// vtctl WaitForFilteredReplication fails if the calculated delay is negative. -func TestWaitForFilteredReplication_unsyncClocks(t *testing.T) { - target := &pbq.Target{Keyspace: keyspace, Shard: destShard, TabletType: pbt.TabletType_MASTER} - - // Replication is lagging behind. - oneHourDelay := &pbq.RealtimeStats{ - SecondsBehindMasterFilteredReplication: 3600, - } - - // Receiving master's clock is running one hour ahead of the sending master. - negativeDelayFunc := func() *pbq.RealtimeStats { - return &pbq.RealtimeStats{ - SecondsBehindMasterFilteredReplication: -3600, - } - } - - waitForFilteredReplication(t, target, "cannot reliably wait for the filtered replication to catch up", oneHourDelay, negativeDelayFunc) -} - -// TestWaitForFilteredReplication_unsyncClocksTolerance tests that -// vtctl WaitForFilteredReplication succeeds as long as the calculated -// negative delay is above a certain tolerance. -func TestWaitForFilteredReplication_unsyncClocksTolerance(t *testing.T) { - target := &pbq.Target{Keyspace: keyspace, Shard: destShard, TabletType: pbt.TabletType_MASTER} - - // Replication is lagging behind. - oneHourDelay := &pbq.RealtimeStats{ - SecondsBehindMasterFilteredReplication: 3600, - } - - // Tablet is a second ahead of the local clock. - slightNegativeDelayFunc := func() *pbq.RealtimeStats { - return &pbq.RealtimeStats{ - SecondsBehindMasterFilteredReplication: -1, - } - } - - waitForFilteredReplication(t, target, "", oneHourDelay, slightNegativeDelayFunc) -} - func waitForFilteredReplicationDefaultDelay(t *testing.T, target *pbq.Target, expectedErr string) { // Replication is lagging behind. oneHourDelay := &pbq.RealtimeStats{ From 3428cd2e61b302ea215c6df32b92e6e3cf1b0d00 Mon Sep 17 00:00:00 2001 From: Alain Jobart Date: Tue, 18 Aug 2015 07:45:36 -0700 Subject: [PATCH 34/80] Changing WatchEndPoints into WatchSrvKeyspace. --- go/vt/etcdtopo/server_test.go | 4 +- go/vt/etcdtopo/serving_graph.go | 41 +++++++++----- go/vt/topo/helpers/tee.go | 6 +- go/vt/topo/helpers/tee_topo_test.go | 4 +- go/vt/topo/server.go | 10 ++-- go/vt/topo/test/faketopo/faketopo.go | 4 +- go/vt/topo/test/serving.go | 84 ++++++++++++++-------------- go/vt/zktopo/serving_graph.go | 20 ++++--- go/vt/zktopo/zktopo_test.go | 4 +- 9 files changed, 95 insertions(+), 82 deletions(-) diff --git a/go/vt/etcdtopo/server_test.go b/go/vt/etcdtopo/server_test.go index b5f8637005..0075bc6b81 100644 --- a/go/vt/etcdtopo/server_test.go +++ b/go/vt/etcdtopo/server_test.go @@ -65,11 +65,11 @@ func TestServingGraph(t *testing.T) { test.CheckServingGraph(ctx, t, ts) } -func TestWatchEndPoints(t *testing.T) { +func TestWatchSrvKeyspace(t *testing.T) { ctx := context.Background() ts := newTestServer(t, []string{"test"}) defer ts.Close() - test.CheckWatchEndPoints(ctx, t, ts) + test.CheckWatchSrvKeyspace(ctx, t, ts) } func TestKeyspaceLock(t *testing.T) { diff --git a/go/vt/etcdtopo/serving_graph.go b/go/vt/etcdtopo/serving_graph.go index ecd3825e96..a158dd8d5a 100644 --- a/go/vt/etcdtopo/serving_graph.go +++ b/go/vt/etcdtopo/serving_graph.go @@ -271,15 +271,15 @@ func (s *Server) GetSrvKeyspaceNames(ctx context.Context, cellName string) ([]st return getNodeNames(resp) } -// WatchEndPoints is part of the topo.Server interface -func (s *Server) WatchEndPoints(ctx context.Context, cellName, keyspace, shard string, tabletType pb.TabletType) (<-chan *pb.EndPoints, chan<- struct{}, error) { +// WatchSrvKeyspace is part of the topo.Server interface +func (s *Server) WatchSrvKeyspace(ctx context.Context, cellName, keyspace string) (<-chan *topo.SrvKeyspace, chan<- struct{}, error) { cell, err := s.getCell(cellName) if err != nil { - return nil, nil, fmt.Errorf("WatchEndPoints cannot get cell: %v", err) + return nil, nil, fmt.Errorf("WatchSrvKeyspace cannot get cell: %v", err) } - filePath := endPointsFilePath(keyspace, shard, tabletType) + filePath := srvKeyspaceFilePath(keyspace) - notifications := make(chan *pb.EndPoints, 10) + notifications := make(chan *topo.SrvKeyspace, 10) stopWatching := make(chan struct{}) // The watch go routine will stop if the 'stop' channel is closed. @@ -288,12 +288,22 @@ func (s *Server) WatchEndPoints(ctx context.Context, cellName, keyspace, shard s watch := make(chan *etcd.Response) stop := make(chan bool) go func() { - // get the current version of the file - ep, modifiedVersion, err := s.GetEndPoints(ctx, cellName, keyspace, shard, tabletType) - if err != nil { + var srvKeyspace *topo.SrvKeyspace + var modifiedVersion int64 + + resp, err := cell.Get(filePath, false /* sort */, false /* recursive */) + if err != nil || resp.Node == nil { // node doesn't exist - modifiedVersion = 0 - ep = nil + } else { + if resp.Node.Value != "" { + sk := &pb.SrvKeyspace{} + if err := json.Unmarshal([]byte(resp.Node.Value), sk); err != nil { + log.Warningf("bad SrvKeyspace data (%v): %q", err, resp.Node.Value) + } else { + srvKeyspace = topo.ProtoToSrvKeyspace(sk) + modifiedVersion = int64(resp.Node.ModifiedIndex) + } + } } // re-check for stop here to be safe, in case the @@ -301,7 +311,7 @@ func (s *Server) WatchEndPoints(ctx context.Context, cellName, keyspace, shard s select { case <-stop: return - case notifications <- ep: + case notifications <- srvKeyspace: } for { @@ -325,15 +335,16 @@ func (s *Server) WatchEndPoints(ctx context.Context, cellName, keyspace, shard s for { select { case resp := <-watch: - var ep *pb.EndPoints + var srvKeyspace *topo.SrvKeyspace if resp.Node != nil && resp.Node.Value != "" { - ep = &pb.EndPoints{} - if err := json.Unmarshal([]byte(resp.Node.Value), ep); err != nil { + sk := &pb.SrvKeyspace{} + if err := json.Unmarshal([]byte(resp.Node.Value), sk); err != nil { log.Errorf("failed to Unmarshal EndPoints for %v: %v", filePath, err) continue } + srvKeyspace = topo.ProtoToSrvKeyspace(sk) } - notifications <- ep + notifications <- srvKeyspace case <-stopWatching: close(stop) close(notifications) diff --git a/go/vt/topo/helpers/tee.go b/go/vt/topo/helpers/tee.go index feabf42a70..b73d1d7ff2 100644 --- a/go/vt/topo/helpers/tee.go +++ b/go/vt/topo/helpers/tee.go @@ -692,10 +692,10 @@ func (tee *Tee) GetSrvKeyspaceNames(ctx context.Context, cell string) ([]string, return tee.readFrom.GetSrvKeyspaceNames(ctx, cell) } -// WatchEndPoints is part of the topo.Server interface. +// WatchSrvKeyspace is part of the topo.Server interface. // We only watch for changes on the primary. -func (tee *Tee) WatchEndPoints(ctx context.Context, cell, keyspace, shard string, tabletType pb.TabletType) (<-chan *pb.EndPoints, chan<- struct{}, error) { - return tee.primary.WatchEndPoints(ctx, cell, keyspace, shard, tabletType) +func (tee *Tee) WatchSrvKeyspace(ctx context.Context, cell, keyspace string) (<-chan *topo.SrvKeyspace, chan<- struct{}, error) { + return tee.primary.WatchSrvKeyspace(ctx, cell, keyspace) } // diff --git a/go/vt/topo/helpers/tee_topo_test.go b/go/vt/topo/helpers/tee_topo_test.go index f37b7a5654..1650110c39 100644 --- a/go/vt/topo/helpers/tee_topo_test.go +++ b/go/vt/topo/helpers/tee_topo_test.go @@ -72,10 +72,10 @@ func TestServingGraph(t *testing.T) { test.CheckServingGraph(ctx, t, ts) } -func TestWatchEndPoints(t *testing.T) { +func TestWatchSrvKeyspace(t *testing.T) { zktopo.WatchSleepDuration = 2 * time.Millisecond ts := newFakeTeeServer(t) - test.CheckWatchEndPoints(context.Background(), t, ts) + test.CheckWatchSrvKeyspace(context.Background(), t, ts) } func TestShardReplication(t *testing.T) { diff --git a/go/vt/topo/server.go b/go/vt/topo/server.go index 1dab056cea..31022f5a58 100644 --- a/go/vt/topo/server.go +++ b/go/vt/topo/server.go @@ -238,12 +238,12 @@ type Server interface { // Can return ErrNoNode or ErrBadVersion. DeleteEndPoints(ctx context.Context, cell, keyspace, shard string, tabletType pb.TabletType, existingVersion int64) error - // WatchEndPoints returns a channel that receives notifications - // every time EndPoints for the given type changes. + // WatchSrvKeyspace returns a channel that receives notifications + // every time the SrvKeyspace for the given keyspace / cell changes. // It should receive a notification with the initial value fairly - // quickly after this is set. A value of nil means the Endpoints + // quickly after this is set. A value of nil means the SrvKeyspace // object doesn't exist or is empty. To stop watching this - // EndPoints object, close the stopWatching channel. + // SrvKeyspace object, close the stopWatching channel. // If the underlying topo.Server encounters an error watching the node, // it should retry on a regular basis until it can succeed. // The initial error returned by this method is meant to catch @@ -251,7 +251,7 @@ type Server interface { // that are never going to work. Mutiple notifications with the // same contents may be sent (for instance when the serving graph // is rebuilt, but the content hasn't changed). - WatchEndPoints(ctx context.Context, cell, keyspace, shard string, tabletType pb.TabletType) (notifications <-chan *pb.EndPoints, stopWatching chan<- struct{}, err error) + WatchSrvKeyspace(ctx context.Context, cell, keyspace string) (notifications <-chan *SrvKeyspace, stopWatching chan<- struct{}, err error) // UpdateSrvShard updates the serving records for a cell, // keyspace, shard. diff --git a/go/vt/topo/test/faketopo/faketopo.go b/go/vt/topo/test/faketopo/faketopo.go index 48ddc657b8..5710d1b8ef 100644 --- a/go/vt/topo/test/faketopo/faketopo.go +++ b/go/vt/topo/test/faketopo/faketopo.go @@ -179,8 +179,8 @@ func (ft FakeTopo) DeleteEndPoints(ctx context.Context, cell, keyspace, shard st return errNotImplemented } -// WatchEndPoints implements topo.Server. -func (ft FakeTopo) WatchEndPoints(ctx context.Context, cell, keyspace, shard string, tabletType pb.TabletType) (<-chan *pb.EndPoints, chan<- struct{}, error) { +// WatchSrvKeyspace implements topo.Server.WatchSrvKeyspace +func (ft FakeTopo) WatchSrvKeyspace(ctx context.Context, cell, keyspace string) (<-chan *topo.SrvKeyspace, chan<- struct{}, error) { return nil, nil, errNotImplemented } diff --git a/go/vt/topo/test/serving.go b/go/vt/topo/test/serving.go index 275eb309c2..523bf1bd95 100644 --- a/go/vt/topo/test/serving.go +++ b/go/vt/topo/test/serving.go @@ -212,93 +212,93 @@ func CheckServingGraph(ctx context.Context, t *testing.T, ts topo.Server) { } } -// CheckWatchEndPoints makes sure WatchEndPoints works as expected -func CheckWatchEndPoints(ctx context.Context, t *testing.T, ts topo.Server) { +// CheckWatchSrvKeyspace makes sure WatchSrvKeyspace works as expected +func CheckWatchSrvKeyspace(ctx context.Context, t *testing.T, ts topo.Server) { cell := getLocalCell(ctx, t, ts) keyspace := "test_keyspace" - shard := "-10" - tabletType := pb.TabletType_MASTER // start watching, should get nil first - notifications, stopWatching, err := ts.WatchEndPoints(ctx, cell, keyspace, shard, tabletType) + notifications, stopWatching, err := ts.WatchSrvKeyspace(ctx, cell, keyspace) if err != nil { - t.Fatalf("WatchEndPoints failed: %v", err) + t.Fatalf("WatchSrvKeyspace failed: %v", err) } - ep, ok := <-notifications - if !ok || ep != nil { - t.Fatalf("first value is wrong: %v %v", ep, ok) + sk, ok := <-notifications + if !ok || sk != nil { + t.Fatalf("first value is wrong: %v %v", sk, ok) } - // update the endpoints, should get a notification - endPoints := &pb.EndPoints{ - Entries: []*pb.EndPoint{ - &pb.EndPoint{ - Uid: 1, - Host: "host1", - PortMap: map[string]int32{ - "vt": 1234, - "mysql": 1235, - "grpc": 1236, + // update the SrvKeyspace, should get a notification + srvKeyspace := &topo.SrvKeyspace{ + ShardingColumnName: "test_column", + Partitions: map[topo.TabletType]*topo.KeyspacePartition{ + topo.TYPE_RDONLY: &topo.KeyspacePartition{ + ShardReferences: []topo.ShardReference{ + topo.ShardReference{ + Name: "0", + }, }, }, }, + ServedFrom: map[topo.TabletType]string{ + topo.TYPE_MASTER: "other_keyspace", + }, } - if err := topo.UpdateEndPoints(ctx, ts, cell, keyspace, shard, tabletType, endPoints, -1); err != nil { - t.Fatalf("UpdateEndPoints failed: %v", err) + if err := ts.UpdateSrvKeyspace(ctx, cell, keyspace, srvKeyspace); err != nil { + t.Fatalf("UpdateSrvKeyspace failed: %v", err) } for { - ep, ok := <-notifications + sk, ok := <-notifications if !ok { t.Fatalf("watch channel is closed???") } - if ep == nil { + if sk == nil { // duplicate notification of the first value, that's OK continue } // non-empty value, that one should be ours - if !reflect.DeepEqual(endPoints, ep) { - t.Fatalf("first value is wrong: %v %v", ep, ok) + if !reflect.DeepEqual(sk, srvKeyspace) { + t.Fatalf("first value is wrong: got %v expected %v", sk, srvKeyspace) } break } - // delete the endpoints, should get a notification - if err := ts.DeleteEndPoints(ctx, cell, keyspace, shard, tabletType, -1); err != nil { - t.Fatalf("DeleteEndPoints failed: %v", err) + // delete the SrvKeyspace, should get a notification + if err := ts.DeleteSrvKeyspace(ctx, cell, keyspace); err != nil { + t.Fatalf("DeleteSrvKeyspace failed: %v", err) } for { - ep, ok := <-notifications + sk, ok := <-notifications if !ok { t.Fatalf("watch channel is closed???") } - if ep == nil { + if sk == nil { break } // duplicate notification of the first value, that's OK, // but value better be good. - if !reflect.DeepEqual(endPoints, ep) { - t.Fatalf("duplicate notification value is bad: %v", ep) + if !reflect.DeepEqual(srvKeyspace, sk) { + t.Fatalf("duplicate notification value is bad: %v", sk) } } // re-create the value, a bit different, should get a notification - endPoints.Entries[0].Uid = 2 - if err := topo.UpdateEndPoints(ctx, ts, cell, keyspace, shard, tabletType, endPoints, -1); err != nil { - t.Fatalf("UpdateEndPoints failed: %v", err) + srvKeyspace.SplitShardCount = 2 + if err := ts.UpdateSrvKeyspace(ctx, cell, keyspace, srvKeyspace); err != nil { + t.Fatalf("UpdateSrvKeyspace failed: %v", err) } for { - ep, ok := <-notifications + sk, ok := <-notifications if !ok { t.Fatalf("watch channel is closed???") } - if ep == nil { + if sk == nil { // duplicate notification of the closed value, that's OK continue } // non-empty value, that one should be ours - if !reflect.DeepEqual(endPoints, ep) { - t.Fatalf("value after delete / re-create is wrong: %v %v", ep, ok) + if !reflect.DeepEqual(srvKeyspace, sk) { + t.Fatalf("value after delete / re-create is wrong: %v %v", sk, ok) } break } @@ -307,12 +307,12 @@ func CheckWatchEndPoints(ctx context.Context, t *testing.T, ts topo.Server) { // notifications channel too close(stopWatching) for { - ep, ok := <-notifications + sk, ok := <-notifications if !ok { break } - if !reflect.DeepEqual(endPoints, ep) { - t.Fatalf("duplicate notification value is bad: %v", ep) + if !reflect.DeepEqual(srvKeyspace, sk) { + t.Fatalf("duplicate notification value is bad: %v", sk) } } } diff --git a/go/vt/zktopo/serving_graph.go b/go/vt/zktopo/serving_graph.go index 1b02153cd4..9c2e1d516f 100644 --- a/go/vt/zktopo/serving_graph.go +++ b/go/vt/zktopo/serving_graph.go @@ -312,11 +312,11 @@ func (zkts *Server) updateTabletEndpoint(oldValue string, oldStat zk.Stat, addr return string(data), nil } -// WatchEndPoints is part of the topo.Server interface -func (zkts *Server) WatchEndPoints(ctx context.Context, cell, keyspace, shard string, tabletType pb.TabletType) (<-chan *pb.EndPoints, chan<- struct{}, error) { - filePath := zkPathForVtName(cell, keyspace, shard, tabletType) +// WatchSrvKeyspace is part of the topo.Server interface +func (zkts *Server) WatchSrvKeyspace(ctx context.Context, cell, keyspace string) (<-chan *topo.SrvKeyspace, chan<- struct{}, error) { + filePath := zkPathForVtKeyspace(cell, keyspace) - notifications := make(chan *pb.EndPoints, 10) + notifications := make(chan *topo.SrvKeyspace, 10) stopWatching := make(chan struct{}) // waitOrInterrupted will return true if stopWatching is triggered @@ -350,17 +350,19 @@ func (zkts *Server) WatchEndPoints(ctx context.Context, cell, keyspace, shard st // get the initial value, send it, or send nil if no // data - var ep *pb.EndPoints + var srvKeyspace *topo.SrvKeyspace sendIt := true if len(data) > 0 { - ep = &pb.EndPoints{} - if err := json.Unmarshal([]byte(data), ep); err != nil { - log.Errorf("EndPoints unmarshal failed: %v %v", data, err) + sk := &pb.SrvKeyspace{} + if err := json.Unmarshal([]byte(data), sk); err != nil { + log.Errorf("SrvKeyspace unmarshal failed: %v %v", data, err) sendIt = false + } else { + srvKeyspace = topo.ProtoToSrvKeyspace(sk) } } if sendIt { - notifications <- ep + notifications <- srvKeyspace } // now act on the watch diff --git a/go/vt/zktopo/zktopo_test.go b/go/vt/zktopo/zktopo_test.go index 81329dfb76..4f21da4191 100644 --- a/go/vt/zktopo/zktopo_test.go +++ b/go/vt/zktopo/zktopo_test.go @@ -48,11 +48,11 @@ func TestServingGraph(t *testing.T) { test.CheckServingGraph(context.Background(), t, ts) } -func TestWatchEndPoints(t *testing.T) { +func TestWatchSrvKeyspace(t *testing.T) { WatchSleepDuration = 2 * time.Millisecond ts := NewTestServer(t, []string{"test"}) defer ts.Close() - test.CheckWatchEndPoints(context.Background(), t, ts) + test.CheckWatchSrvKeyspace(context.Background(), t, ts) } func TestKeyspaceLock(t *testing.T) { From 81571296b87cbfc4a251715684af46934070aaf9 Mon Sep 17 00:00:00 2001 From: Alain Jobart Date: Tue, 18 Aug 2015 08:09:56 -0700 Subject: [PATCH 35/80] Making vertical_split.py always and only work with vtgate. --- Makefile | 2 - test/config.json | 3 - test/vertical_split.py | 112 ++++++++++++---------------------- test/vertical_split_vtgate.py | 67 -------------------- 4 files changed, 40 insertions(+), 144 deletions(-) delete mode 100755 test/vertical_split_vtgate.py diff --git a/Makefile b/Makefile index d62da6e61f..fd708de40f 100644 --- a/Makefile +++ b/Makefile @@ -97,7 +97,6 @@ small_integration_test_files = \ tablet_test.py \ sql_builder_test.py \ vertical_split.py \ - vertical_split_vtgate.py \ schema.py \ keyspace_test.py \ keyrange_test.py \ @@ -138,7 +137,6 @@ worker_integration_test_files = \ resharding.py \ resharding_bytes.py \ vertical_split.py \ - vertical_split_vtgate.py \ initial_sharding.py \ initial_sharding_bytes.py \ worker.py diff --git a/test/config.json b/test/config.json index 7cf5a9ac49..a4ceef5186 100644 --- a/test/config.json +++ b/test/config.json @@ -17,9 +17,6 @@ "vertical_split": { "File": "vertical_split.py" }, - "vertical_split_vtgate": { - "File": "vertical_split_vtgate.py" - }, "schema": { "File": "schema.py" }, diff --git a/test/vertical_split.py b/test/vertical_split.py index daa54271af..e205c7bc4d 100755 --- a/test/vertical_split.py +++ b/test/vertical_split.py @@ -5,24 +5,17 @@ # be found in the LICENSE file. import logging -import threading -import struct import time import unittest -from zk import zkocc -from vtdb import dbexceptions -from vtdb import topology -from vtdb import vtclient +from vtdb import keyrange +from vtdb import keyrange_constants +from vtdb import vtgate_client import environment import utils import tablet - -TABLET = 'tablet' -VTGATE = 'vtgate' -VTGATE_PROTOCOL_TABLET = 'v0' -client_type = TABLET +from protocols_flavor import protocols_flavor # source keyspace, with 4 tables source_master = tablet.Tablet() @@ -51,6 +44,7 @@ def setUpModule(): destination_rdonly2.init_mysql(), ] utils.Vtctld().start() + utils.VtGate().start(cache_ttl='0s') utils.wait_procs(setup_procs) except: tearDownModule() @@ -61,6 +55,7 @@ def tearDownModule(): if utils.options.skip_teardown: return + utils.vtgate.kill() teardown_procs = [ source_master.teardown_mysql(), source_replica.teardown_mysql(), @@ -88,21 +83,7 @@ def tearDownModule(): class TestVerticalSplit(unittest.TestCase): def setUp(self): - utils.VtGate().start(cache_ttl='0s') - self.vtgate_client = zkocc.ZkOccConnection(utils.vtgate.addr(), - 'test_nj', 30.0) - self.vtgate_addrs = None - if client_type == VTGATE: - self.vtgate_addrs = {'vt': [utils.vtgate.addr(),]} - self.insert_index = 0 - # Lowering the keyspace refresh throttle so things are testable. - self.throttle_sleep_interval = 0.1 - topology.set_keyspace_fetch_throttle(0.01) - - def tearDown(self): - self.vtgate_client.close() - utils.vtgate.kill() def _create_source_schema(self): create_table_template = '''create table %s( @@ -123,31 +104,23 @@ index by_msg (msg) 'source_keyspace'], auto_log=True) - def _vtdb_conn(self, db_type='master', keyspace='source_keyspace'): - global client_type - vtgate_protocol = None - if self.vtgate_addrs is None: - self.vtgate_addrs = {} - if client_type == TABLET: - vtgate_protocol = VTGATE_PROTOCOL_TABLET - conn = vtclient.VtOCCConnection(self.vtgate_client, keyspace, '0', - db_type, 30, - vtgate_protocol=vtgate_protocol, - vtgate_addrs=self.vtgate_addrs) - conn.connect() - return conn + def _vtdb_conn(self): + addr = utils.vtgate.rpc_endpoint() + protocol = protocols_flavor().vtgate_python_protocol() + return vtgate_client.connect(protocol, addr, 30.0) # insert some values in the source master db, return the first id used def _insert_values(self, table, count): result = self.insert_index + conn = self._vtdb_conn() + cursor = conn.cursor('source_keyspace', 'master', keyranges=[keyrange.KeyRange(keyrange_constants.NON_PARTIAL_KEYRANGE)], writable=True) for i in xrange(count): - source_master.execute('insert into %s (id, msg) values(:id, :msg)' % - table, - bindvars={ - 'id': self.insert_index, - 'msg': 'value %d' % self.insert_index, - }) + conn.begin() + cursor.execute("insert into %s (id, msg) values(%d, 'value %d')" % ( + table, self.insert_index, self.insert_index), {}) + conn.commit() self.insert_index += 1 + conn.close() return result def _check_values(self, tablet, dbname, table, first, count): @@ -222,31 +195,31 @@ index by_msg (msg) logging.debug('Got %s rows from table %s on tablet %s', qr['Rows'][0][0], t, tablet.tablet_alias) - def _populate_topo_cache(self): - topology.read_topology(self.vtgate_client) - - def refresh_keyspace(self, keyspace_name): - # This is so that keyspace can be refreshed. - time.sleep(self.throttle_sleep_interval) - topology.refresh_keyspace(self.vtgate_client, keyspace_name) - - def _check_client_conn_redirection(self, source_ks, destination_ks, db_types, - servedfrom_db_types, moved_tables=None): - # In normal operations, it takes the first error for keyspace to be re-read. - # For testing purposes, refreshing the topology manually. - self.refresh_keyspace(destination_ks) - + def _check_client_conn_redirection(self, source_ks, destination_ks, db_types, servedfrom_db_types, moved_tables=None): + # check that the ServedFrom indirection worked correctly. + if moved_tables is None: + moved_tables = [] + conn = self._vtdb_conn() for db_type in servedfrom_db_types: - conn = self._vtdb_conn(db_type, keyspace=destination_ks) - self.assertEqual(conn.db_params['keyspace'], source_ks) - - # check that the connection to db_type for destination keyspace works too. - for db_type in db_types: - dest_conn = self._vtdb_conn(db_type, keyspace=destination_ks) - self.assertEqual(dest_conn.db_params['keyspace'], destination_ks) + for tbl in moved_tables: + try: + rows = conn._execute("select * from %s" % tbl, {}, destination_ks, db_type, keyranges=[keyrange.KeyRange(keyrange_constants.NON_PARTIAL_KEYRANGE)]) + logging.debug("Select on %s.%s returned %d rows" % (db_type, tbl, len(rows))) + except Exception, e: + self.fail("Execute failed w/ exception %s" % str(e)) def _check_stats(self): - pass + v = utils.vtgate.get_vars() + self.assertEqual(v['VttabletCall']['Histograms']['Execute.source_keyspace.0.replica']['Count'], 2, "unexpected value for VttabletCall(Execute.source_keyspace.0.replica) inside %s" % str(v)) + self.assertEqual(v['VtgateApi']['Histograms']['ExecuteKeyRanges.destination_keyspace.master']['Count'], 6, "unexpected value for VtgateApi(ExecuteKeyRanges.destination_keyspace.master) inside %s" % str(v)) + self.assertEqual(len(v['VtgateApiErrorCounts']), 0, "unexpected errors for VtgateApiErrorCounts inside %s" % str(v)) + self.assertEqual( + v['ResilientSrvTopoServerEndPointsReturnedCount']['test_nj.source_keyspace.0.master'] / + v['ResilientSrvTopoServerEndPointQueryCount']['test_nj.source_keyspace.0.master'], + 1, "unexpected EndPointsReturnedCount inside %s" % str(v)) + self.assertNotIn( + 'test_nj.source_keyspace.0.master', v['ResilientSrvTopoServerEndPointDegradedResultCount'], + "unexpected EndPointDegradedResultCount inside %s" % str(v)) def test_vertical_split(self): utils.run_vtctl(['CreateKeyspace', 'source_keyspace']) @@ -301,9 +274,6 @@ index by_msg (msg) utils.run_vtctl(['InitShardMaster', 'destination_keyspace/0', destination_master.tablet_alias], auto_log=True) - # read all the keyspaces, this will populate the topology cache. - self._populate_topo_cache() - # create the schema on the source keyspace, add some values self._create_source_schema() moving1_first = self._insert_values('moving1', 100) @@ -343,8 +313,6 @@ index by_msg (msg) utils.run_vtctl(['ChangeSlaveType', source_rdonly2.tablet_alias, 'rdonly'], auto_log=True) - topology.refresh_keyspace(self.vtgate_client, 'destination_keyspace') - # check values are present self._check_values(destination_master, 'vt_destination_keyspace', 'moving1', moving1_first, 100) @@ -522,7 +490,7 @@ index by_msg (msg) # check the binlog player is gone now destination_master.wait_for_binlog_player_count(0) - # optional method to check the stats are correct + # check the stats are correct self._check_stats() # kill everything diff --git a/test/vertical_split_vtgate.py b/test/vertical_split_vtgate.py deleted file mode 100755 index 277a3f1b7f..0000000000 --- a/test/vertical_split_vtgate.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2013, Google Inc. All rights reserved. -# Use of this source code is governed by a BSD-style license that can -# be found in the LICENSE file. - -import logging -import vertical_split -import utils - -from vtdb import keyrange -from vtdb import keyrange_constants -from vtdb import vtgatev2 - -def setUpModule(): - vertical_split.setUpModule() - -def tearDownModule(): - vertical_split.tearDownModule() - -class TestVerticalSplitVTGate(vertical_split.TestVerticalSplit): - def _vtdb_conn(self): - conn = vtgatev2.connect(self.vtgate_addrs['vt'], 30) - return conn - - def _insert_values(self, table, count, db_type='master', keyspace='source_keyspace'): - result = self.insert_index - conn = self._vtdb_conn() - cursor = conn.cursor(keyspace, db_type, keyranges=[keyrange.KeyRange(keyrange_constants.NON_PARTIAL_KEYRANGE)], writable=True) - for i in xrange(count): - conn.begin() - cursor.execute("insert into %s (id, msg) values(%d, 'value %d')" % ( - table, self.insert_index, self.insert_index), {}) - conn.commit() - self.insert_index += 1 - conn.close() - return result - - def _check_client_conn_redirection(self, source_ks, destination_ks, db_types, servedfrom_db_types, moved_tables=None): - # check that the ServedFrom indirection worked correctly. - if moved_tables is None: - moved_tables = [] - conn = self._vtdb_conn() - for db_type in servedfrom_db_types: - for tbl in moved_tables: - try: - rows = conn._execute("select * from %s" % tbl, {}, destination_ks, db_type, keyranges=[keyrange.KeyRange(keyrange_constants.NON_PARTIAL_KEYRANGE)]) - logging.debug("Select on %s.%s returned %d rows" % (db_type, tbl, len(rows))) - except Exception, e: - self.fail("Execute failed w/ exception %s" % str(e)) - - def _check_stats(self): - v = utils.vtgate.get_vars() - self.assertEqual(v['VttabletCall']['Histograms']['Execute.source_keyspace.0.replica']['Count'], 2, "unexpected value for VttabletCall(Execute.source_keyspace.0.replica) inside %s" % str(v)) - self.assertEqual(v['VtgateApi']['Histograms']['ExecuteKeyRanges.destination_keyspace.master']['Count'], 6, "unexpected value for VtgateApi(ExecuteKeyRanges.destination_keyspace.master) inside %s" % str(v)) - self.assertEqual(len(v['VtgateApiErrorCounts']), 0, "unexpected errors for VtgateApiErrorCounts inside %s" % str(v)) - self.assertEqual( - v['ResilientSrvTopoServerEndPointsReturnedCount']['test_nj.source_keyspace.0.master'] / - v['ResilientSrvTopoServerEndPointQueryCount']['test_nj.source_keyspace.0.master'], - 1, "unexpected EndPointsReturnedCount inside %s" % str(v)) - self.assertNotIn( - 'test_nj.source_keyspace.0.master', v['ResilientSrvTopoServerEndPointDegradedResultCount'], - "unexpected EndPointDegradedResultCount inside %s" % str(v)) - -if __name__ == '__main__': - vertical_split.client_type = vertical_split.VTGATE - utils.main() From 4538a917bce97ee483a96174f4f52043d1adf919 Mon Sep 17 00:00:00 2001 From: Alain Jobart Date: Tue, 18 Aug 2015 11:39:43 -0700 Subject: [PATCH 36/80] New version of protobuf / grpc libraries. --- .../automationservice/automationservice.pb.go | 6 +- go/vt/proto/binlogservice/binlogservice.pb.go | 6 +- go/vt/proto/mysqlctl/mysqlctl.pb.go | 8 +- go/vt/proto/queryservice/queryservice.pb.go | 6 +- .../tabletmanagerservice.pb.go | 6 +- go/vt/proto/vtctlservice/vtctlservice.pb.go | 6 +- go/vt/proto/vtgateservice/vtgateservice.pb.go | 6 +- .../vtworkerservice/vtworkerservice.pb.go | 6 +- py/vtproto/automation_pb2.py | 12 +++ py/vtproto/automationservice_pb2.py | 9 +- py/vtproto/binlogdata_pb2.py | 11 +++ py/vtproto/binlogservice_pb2.py | 9 +- py/vtproto/logutil_pb2.py | 3 + py/vtproto/mysqlctl_pb2.py | 15 ++- py/vtproto/query_pb2.py | 29 ++++++ py/vtproto/queryservice_pb2.py | 9 +- py/vtproto/replicationdata_pb2.py | 2 + py/vtproto/tableacl_pb2.py | 3 + py/vtproto/tabletmanagerdata_pb2.py | 92 +++++++++++++++++++ py/vtproto/tabletmanagerservice_pb2.py | 9 +- py/vtproto/topodata_pb2.py | 24 +++++ py/vtproto/vtctldata_pb2.py | 3 + py/vtproto/vtctlservice_pb2.py | 9 +- py/vtproto/vtgate_pb2.py | 41 +++++++++ py/vtproto/vtgateservice_pb2.py | 9 +- py/vtproto/vtrpc_pb2.py | 3 + py/vtproto/vtworkerdata_pb2.py | 3 + py/vtproto/vtworkerservice_pb2.py | 9 +- travis/install_grpc.sh | 2 +- travis/install_protobuf.sh | 6 +- 30 files changed, 301 insertions(+), 61 deletions(-) diff --git a/go/vt/proto/automationservice/automationservice.pb.go b/go/vt/proto/automationservice/automationservice.pb.go index d08d57fcb3..cdf46b8e5b 100644 --- a/go/vt/proto/automationservice/automationservice.pb.go +++ b/go/vt/proto/automationservice/automationservice.pb.go @@ -21,11 +21,11 @@ import ( ) // Reference imports to suppress errors if they are not otherwise used. -var _ context.Context -var _ grpc.ClientConn +var _ = proto.Marshal // Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal +var _ context.Context +var _ grpc.ClientConn // Client API for Automation service diff --git a/go/vt/proto/binlogservice/binlogservice.pb.go b/go/vt/proto/binlogservice/binlogservice.pb.go index b56ae033dd..23eecadc36 100644 --- a/go/vt/proto/binlogservice/binlogservice.pb.go +++ b/go/vt/proto/binlogservice/binlogservice.pb.go @@ -21,11 +21,11 @@ import ( ) // Reference imports to suppress errors if they are not otherwise used. -var _ context.Context -var _ grpc.ClientConn +var _ = proto.Marshal // Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal +var _ context.Context +var _ grpc.ClientConn // Client API for UpdateStream service diff --git a/go/vt/proto/mysqlctl/mysqlctl.pb.go b/go/vt/proto/mysqlctl/mysqlctl.pb.go index b90d8f3361..9e524ab65f 100644 --- a/go/vt/proto/mysqlctl/mysqlctl.pb.go +++ b/go/vt/proto/mysqlctl/mysqlctl.pb.go @@ -25,10 +25,6 @@ import ( grpc "google.golang.org/grpc" ) -// Reference imports to suppress errors if they are not otherwise used. -var _ context.Context -var _ grpc.ClientConn - // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal @@ -75,6 +71,10 @@ func (m *RunMysqlUpgradeResponse) Reset() { *m = RunMysqlUpgradeResponse func (m *RunMysqlUpgradeResponse) String() string { return proto.CompactTextString(m) } func (*RunMysqlUpgradeResponse) ProtoMessage() {} +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + // Client API for MysqlCtl service type MysqlCtlClient interface { diff --git a/go/vt/proto/queryservice/queryservice.pb.go b/go/vt/proto/queryservice/queryservice.pb.go index b2966b6351..e733cca3de 100644 --- a/go/vt/proto/queryservice/queryservice.pb.go +++ b/go/vt/proto/queryservice/queryservice.pb.go @@ -21,11 +21,11 @@ import ( ) // Reference imports to suppress errors if they are not otherwise used. -var _ context.Context -var _ grpc.ClientConn +var _ = proto.Marshal // Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal +var _ context.Context +var _ grpc.ClientConn // Client API for Query service diff --git a/go/vt/proto/tabletmanagerservice/tabletmanagerservice.pb.go b/go/vt/proto/tabletmanagerservice/tabletmanagerservice.pb.go index c2ff586c42..edc7069185 100644 --- a/go/vt/proto/tabletmanagerservice/tabletmanagerservice.pb.go +++ b/go/vt/proto/tabletmanagerservice/tabletmanagerservice.pb.go @@ -21,11 +21,11 @@ import ( ) // Reference imports to suppress errors if they are not otherwise used. -var _ context.Context -var _ grpc.ClientConn +var _ = proto.Marshal // Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal +var _ context.Context +var _ grpc.ClientConn // Client API for TabletManager service diff --git a/go/vt/proto/vtctlservice/vtctlservice.pb.go b/go/vt/proto/vtctlservice/vtctlservice.pb.go index 33ecc6adeb..bdafbf8540 100644 --- a/go/vt/proto/vtctlservice/vtctlservice.pb.go +++ b/go/vt/proto/vtctlservice/vtctlservice.pb.go @@ -21,11 +21,11 @@ import ( ) // Reference imports to suppress errors if they are not otherwise used. -var _ context.Context -var _ grpc.ClientConn +var _ = proto.Marshal // Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal +var _ context.Context +var _ grpc.ClientConn // Client API for Vtctl service diff --git a/go/vt/proto/vtgateservice/vtgateservice.pb.go b/go/vt/proto/vtgateservice/vtgateservice.pb.go index b17d035520..88281c9fe5 100644 --- a/go/vt/proto/vtgateservice/vtgateservice.pb.go +++ b/go/vt/proto/vtgateservice/vtgateservice.pb.go @@ -21,11 +21,11 @@ import ( ) // Reference imports to suppress errors if they are not otherwise used. -var _ context.Context -var _ grpc.ClientConn +var _ = proto.Marshal // Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal +var _ context.Context +var _ grpc.ClientConn // Client API for Vitess service diff --git a/go/vt/proto/vtworkerservice/vtworkerservice.pb.go b/go/vt/proto/vtworkerservice/vtworkerservice.pb.go index 0434258e3a..e4585a2389 100644 --- a/go/vt/proto/vtworkerservice/vtworkerservice.pb.go +++ b/go/vt/proto/vtworkerservice/vtworkerservice.pb.go @@ -21,11 +21,11 @@ import ( ) // Reference imports to suppress errors if they are not otherwise used. -var _ context.Context -var _ grpc.ClientConn +var _ = proto.Marshal // Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal +var _ context.Context +var _ grpc.ClientConn // Client API for Vtworker service diff --git a/py/vtproto/automation_pb2.py b/py/vtproto/automation_pb2.py index d0f4d80ca9..6e670c690e 100644 --- a/py/vtproto/automation_pb2.py +++ b/py/vtproto/automation_pb2.py @@ -19,6 +19,7 @@ _sym_db = _symbol_database.Default() DESCRIPTOR = _descriptor.FileDescriptor( name='automation.proto', package='automation', + syntax='proto3', serialized_pb=_b('\n\x10\x61utomation.proto\x12\nautomation\"\x90\x01\n\x10\x43lusterOperation\x12\n\n\x02id\x18\x01 \x01(\t\x12/\n\x0cserial_tasks\x18\x02 \x03(\x0b\x32\x19.automation.TaskContainer\x12\x30\n\x05state\x18\x03 \x01(\x0e\x32!.automation.ClusterOperationState\x12\r\n\x05\x65rror\x18\x04 \x01(\t\"N\n\rTaskContainer\x12(\n\x0eparallel_tasks\x18\x01 \x03(\x0b\x32\x10.automation.Task\x12\x13\n\x0b\x63oncurrency\x18\x02 \x01(\x05\"\xce\x01\n\x04Task\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x34\n\nparameters\x18\x02 \x03(\x0b\x32 .automation.Task.ParametersEntry\x12\n\n\x02id\x18\x03 \x01(\t\x12$\n\x05state\x18\x04 \x01(\x0e\x32\x15.automation.TaskState\x12\x0e\n\x06output\x18\x05 \x01(\t\x12\r\n\x05\x65rror\x18\x06 \x01(\t\x1a\x31\n\x0fParametersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xb1\x01\n\x1e\x45nqueueClusterOperationRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12N\n\nparameters\x18\x02 \x03(\x0b\x32:.automation.EnqueueClusterOperationRequest.ParametersEntry\x1a\x31\n\x0fParametersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"-\n\x1f\x45nqueueClusterOperationResponse\x12\n\n\x02id\x18\x01 \x01(\t\"-\n\x1fGetClusterOperationStateRequest\x12\n\n\x02id\x18\x01 \x01(\t\"T\n GetClusterOperationStateResponse\x12\x30\n\x05state\x18\x01 \x01(\x0e\x32!.automation.ClusterOperationState\"/\n!GetClusterOperationDetailsRequest\x12\n\n\x02id\x18\x01 \x01(\t\"V\n\"GetClusterOperationDetailsResponse\x12\x30\n\ncluster_op\x18\x02 \x01(\x0b\x32\x1c.automation.ClusterOperation*\x9a\x01\n\x15\x43lusterOperationState\x12#\n\x1fUNKNOWN_CLUSTER_OPERATION_STATE\x10\x00\x12!\n\x1d\x43LUSTER_OPERATION_NOT_STARTED\x10\x01\x12\x1d\n\x19\x43LUSTER_OPERATION_RUNNING\x10\x02\x12\x1a\n\x16\x43LUSTER_OPERATION_DONE\x10\x03*K\n\tTaskState\x12\x16\n\x12UNKNOWN_TASK_STATE\x10\x00\x12\x0f\n\x0bNOT_STARTED\x10\x01\x12\x0b\n\x07RUNNING\x10\x02\x12\x08\n\x04\x44ONE\x10\x03\x62\x06proto3') ) _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -139,6 +140,7 @@ _CLUSTEROPERATION = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -176,6 +178,7 @@ _TASKCONTAINER = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -213,6 +216,7 @@ _TASK_PARAMETERSENTRY = _descriptor.Descriptor( ], options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')), is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -277,6 +281,7 @@ _TASK = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -314,6 +319,7 @@ _ENQUEUECLUSTEROPERATIONREQUEST_PARAMETERSENTRY = _descriptor.Descriptor( ], options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')), is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -350,6 +356,7 @@ _ENQUEUECLUSTEROPERATIONREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -380,6 +387,7 @@ _ENQUEUECLUSTEROPERATIONRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -410,6 +418,7 @@ _GETCLUSTEROPERATIONSTATEREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -440,6 +449,7 @@ _GETCLUSTEROPERATIONSTATERESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -470,6 +480,7 @@ _GETCLUSTEROPERATIONDETAILSREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -500,6 +511,7 @@ _GETCLUSTEROPERATIONDETAILSRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], diff --git a/py/vtproto/automationservice_pb2.py b/py/vtproto/automationservice_pb2.py index 1630523432..aa72d12f9f 100644 --- a/py/vtproto/automationservice_pb2.py +++ b/py/vtproto/automationservice_pb2.py @@ -19,6 +19,7 @@ import automation_pb2 as automation__pb2 DESCRIPTOR = _descriptor.FileDescriptor( name='automationservice.proto', package='automationservice', + syntax='proto3', serialized_pb=_b('\n\x17\x61utomationservice.proto\x12\x11\x61utomationservice\x1a\x10\x61utomation.proto2\x81\x02\n\nAutomation\x12t\n\x17\x45nqueueClusterOperation\x12*.automation.EnqueueClusterOperationRequest\x1a+.automation.EnqueueClusterOperationResponse\"\x00\x12}\n\x1aGetClusterOperationDetails\x12-.automation.GetClusterOperationDetailsRequest\x1a..automation.GetClusterOperationDetailsResponse\"\x00\x62\x06proto3') , dependencies=[automation__pb2.DESCRIPTOR,]) @@ -60,7 +61,7 @@ class EarlyAdopterAutomationStub(object): def GetClusterOperationDetails(self, request): raise NotImplementedError() GetClusterOperationDetails.async = None -def early_adopter_create_Automation_server(servicer, port, root_certificates, key_chain_pairs): +def early_adopter_create_Automation_server(servicer, port, private_key=None, certificate_chain=None): import automation_pb2 import automation_pb2 import automation_pb2 @@ -77,8 +78,8 @@ def early_adopter_create_Automation_server(servicer, port, root_certificates, ke automation_pb2.GetClusterOperationDetailsResponse.SerializeToString, ), } - return implementations.secure_server("automationservice.Automation", method_service_descriptions, port, root_certificates, key_chain_pairs) -def early_adopter_create_Automation_stub(host, port): + return implementations.server("automationservice.Automation", method_service_descriptions, port, private_key=private_key, certificate_chain=certificate_chain) +def early_adopter_create_Automation_stub(host, port, metadata_transformer=None, secure=False, root_certificates=None, private_key=None, certificate_chain=None, server_host_override=None): import automation_pb2 import automation_pb2 import automation_pb2 @@ -93,5 +94,5 @@ def early_adopter_create_Automation_stub(host, port): automation_pb2.GetClusterOperationDetailsResponse.FromString, ), } - return implementations.insecure_stub("automationservice.Automation", method_invocation_descriptions, host, port) + return implementations.stub("automationservice.Automation", method_invocation_descriptions, host, port, metadata_transformer=metadata_transformer, secure=secure, root_certificates=root_certificates, private_key=private_key, certificate_chain=certificate_chain, server_host_override=server_host_override) # @@protoc_insertion_point(module_scope) diff --git a/py/vtproto/binlogdata_pb2.py b/py/vtproto/binlogdata_pb2.py index 81a79f61a1..ff19880372 100644 --- a/py/vtproto/binlogdata_pb2.py +++ b/py/vtproto/binlogdata_pb2.py @@ -20,6 +20,7 @@ import topodata_pb2 as topodata__pb2 DESCRIPTOR = _descriptor.FileDescriptor( name='binlogdata.proto', package='binlogdata', + syntax='proto3', serialized_pb=_b('\n\x10\x62inlogdata.proto\x12\nbinlogdata\x1a\x0bquery.proto\x1a\x0etopodata.proto\"7\n\x07\x43harset\x12\x0e\n\x06\x63lient\x18\x01 \x01(\x05\x12\x0c\n\x04\x63onn\x18\x02 \x01(\x05\x12\x0e\n\x06server\x18\x03 \x01(\x05\"\xf3\x02\n\x11\x42inlogTransaction\x12;\n\nstatements\x18\x01 \x03(\x0b\x32\'.binlogdata.BinlogTransaction.Statement\x12\x11\n\ttimestamp\x18\x02 \x01(\x03\x12\x16\n\x0etransaction_id\x18\x03 \x01(\t\x1a\xf5\x01\n\tStatement\x12\x42\n\x08\x63\x61tegory\x18\x01 \x01(\x0e\x32\x30.binlogdata.BinlogTransaction.Statement.Category\x12$\n\x07\x63harset\x18\x03 \x01(\x0b\x32\x13.binlogdata.Charset\x12\x0b\n\x03sql\x18\x02 \x01(\x0c\"q\n\x08\x43\x61tegory\x12\x13\n\x0f\x42L_UNRECOGNIZED\x10\x00\x12\x0c\n\x08\x42L_BEGIN\x10\x01\x12\r\n\tBL_COMMIT\x10\x02\x12\x0f\n\x0b\x42L_ROLLBACK\x10\x03\x12\n\n\x06\x42L_DML\x10\x04\x12\n\n\x06\x42L_DDL\x10\x05\x12\n\n\x06\x42L_SET\x10\x06\"\x9b\x02\n\x0bStreamEvent\x12\x32\n\x08\x63\x61tegory\x18\x01 \x01(\x0e\x32 .binlogdata.StreamEvent.Category\x12\x12\n\ntable_name\x18\x02 \x01(\t\x12(\n\x12primary_key_fields\x18\x03 \x03(\x0b\x32\x0c.query.Field\x12&\n\x12primary_key_values\x18\x04 \x03(\x0b\x32\n.query.Row\x12\x0b\n\x03sql\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\x03\x12\x16\n\x0etransaction_id\x18\x07 \x01(\t\":\n\x08\x43\x61tegory\x12\n\n\x06SE_ERR\x10\x00\x12\n\n\x06SE_DML\x10\x01\x12\n\n\x06SE_DDL\x10\x02\x12\n\n\x06SE_POS\x10\x03\"\'\n\x13StreamUpdateRequest\x12\x10\n\x08position\x18\x01 \x01(\t\"E\n\x14StreamUpdateResponse\x12-\n\x0cstream_event\x18\x01 \x01(\x0b\x32\x17.binlogdata.StreamEvent\"\xaa\x01\n\x15StreamKeyRangeRequest\x12\x10\n\x08position\x18\x01 \x01(\t\x12\x32\n\x10keyspace_id_type\x18\x02 \x01(\x0e\x32\x18.topodata.KeyspaceIdType\x12%\n\tkey_range\x18\x03 \x01(\x0b\x32\x12.topodata.KeyRange\x12$\n\x07\x63harset\x18\x04 \x01(\x0b\x32\x13.binlogdata.Charset\"S\n\x16StreamKeyRangeResponse\x12\x39\n\x12\x62inlog_transaction\x18\x01 \x01(\x0b\x32\x1d.binlogdata.BinlogTransaction\"]\n\x13StreamTablesRequest\x12\x10\n\x08position\x18\x01 \x01(\t\x12\x0e\n\x06tables\x18\x02 \x03(\t\x12$\n\x07\x63harset\x18\x03 \x01(\x0b\x32\x13.binlogdata.Charset\"Q\n\x14StreamTablesResponse\x12\x39\n\x12\x62inlog_transaction\x18\x01 \x01(\x0b\x32\x1d.binlogdata.BinlogTransactionb\x06proto3') , dependencies=[query__pb2.DESCRIPTOR,topodata__pb2.DESCRIPTOR,]) @@ -136,6 +137,7 @@ _CHARSET = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -181,6 +183,7 @@ _BINLOGTRANSACTION_STATEMENT = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -224,6 +227,7 @@ _BINLOGTRANSACTION = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -297,6 +301,7 @@ _STREAMEVENT = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -327,6 +332,7 @@ _STREAMUPDATEREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -357,6 +363,7 @@ _STREAMUPDATERESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -408,6 +415,7 @@ _STREAMKEYRANGEREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -438,6 +446,7 @@ _STREAMKEYRANGERESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -482,6 +491,7 @@ _STREAMTABLESREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -512,6 +522,7 @@ _STREAMTABLESRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], diff --git a/py/vtproto/binlogservice_pb2.py b/py/vtproto/binlogservice_pb2.py index 92e803ad2e..5c2ff38f9d 100644 --- a/py/vtproto/binlogservice_pb2.py +++ b/py/vtproto/binlogservice_pb2.py @@ -19,6 +19,7 @@ import binlogdata_pb2 as binlogdata__pb2 DESCRIPTOR = _descriptor.FileDescriptor( name='binlogservice.proto', package='binlogservice', + syntax='proto3', serialized_pb=_b('\n\x13\x62inlogservice.proto\x12\rbinlogservice\x1a\x10\x62inlogdata.proto2\x99\x02\n\x0cUpdateStream\x12U\n\x0cStreamUpdate\x12\x1f.binlogdata.StreamUpdateRequest\x1a .binlogdata.StreamUpdateResponse\"\x00\x30\x01\x12[\n\x0eStreamKeyRange\x12!.binlogdata.StreamKeyRangeRequest\x1a\".binlogdata.StreamKeyRangeResponse\"\x00\x30\x01\x12U\n\x0cStreamTables\x12\x1f.binlogdata.StreamTablesRequest\x1a .binlogdata.StreamTablesResponse\"\x00\x30\x01\x62\x06proto3') , dependencies=[binlogdata__pb2.DESCRIPTOR,]) @@ -67,7 +68,7 @@ class EarlyAdopterUpdateStreamStub(object): def StreamTables(self, request): raise NotImplementedError() StreamTables.async = None -def early_adopter_create_UpdateStream_server(servicer, port, root_certificates, key_chain_pairs): +def early_adopter_create_UpdateStream_server(servicer, port, private_key=None, certificate_chain=None): import binlogdata_pb2 import binlogdata_pb2 import binlogdata_pb2 @@ -91,8 +92,8 @@ def early_adopter_create_UpdateStream_server(servicer, port, root_certificates, binlogdata_pb2.StreamUpdateResponse.SerializeToString, ), } - return implementations.secure_server("binlogservice.UpdateStream", method_service_descriptions, port, root_certificates, key_chain_pairs) -def early_adopter_create_UpdateStream_stub(host, port): + return implementations.server("binlogservice.UpdateStream", method_service_descriptions, port, private_key=private_key, certificate_chain=certificate_chain) +def early_adopter_create_UpdateStream_stub(host, port, metadata_transformer=None, secure=False, root_certificates=None, private_key=None, certificate_chain=None, server_host_override=None): import binlogdata_pb2 import binlogdata_pb2 import binlogdata_pb2 @@ -113,5 +114,5 @@ def early_adopter_create_UpdateStream_stub(host, port): binlogdata_pb2.StreamUpdateResponse.FromString, ), } - return implementations.insecure_stub("binlogservice.UpdateStream", method_invocation_descriptions, host, port) + return implementations.stub("binlogservice.UpdateStream", method_invocation_descriptions, host, port, metadata_transformer=metadata_transformer, secure=secure, root_certificates=root_certificates, private_key=private_key, certificate_chain=certificate_chain, server_host_override=server_host_override) # @@protoc_insertion_point(module_scope) diff --git a/py/vtproto/logutil_pb2.py b/py/vtproto/logutil_pb2.py index 7b89b91c5a..766399246c 100644 --- a/py/vtproto/logutil_pb2.py +++ b/py/vtproto/logutil_pb2.py @@ -19,6 +19,7 @@ _sym_db = _symbol_database.Default() DESCRIPTOR = _descriptor.FileDescriptor( name='logutil.proto', package='logutil', + syntax='proto3', serialized_pb=_b('\n\rlogutil.proto\x12\x07logutil\",\n\x04Time\x12\x0f\n\x07seconds\x18\x01 \x01(\x03\x12\x13\n\x0bnanoseconds\x18\x02 \x01(\x05\"n\n\x05\x45vent\x12\x1b\n\x04time\x18\x01 \x01(\x0b\x32\r.logutil.Time\x12\x1d\n\x05level\x18\x02 \x01(\x0e\x32\x0e.logutil.Level\x12\x0c\n\x04\x66ile\x18\x03 \x01(\t\x12\x0c\n\x04line\x18\x04 \x01(\x03\x12\r\n\x05value\x18\x05 \x01(\t*6\n\x05Level\x12\x08\n\x04INFO\x10\x00\x12\x0b\n\x07WARNING\x10\x01\x12\t\n\x05\x45RROR\x10\x02\x12\x0b\n\x07\x43ONSOLE\x10\x03\x62\x06proto3') ) _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -90,6 +91,7 @@ _TIME = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -148,6 +150,7 @@ _EVENT = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], diff --git a/py/vtproto/mysqlctl_pb2.py b/py/vtproto/mysqlctl_pb2.py index f2fd78a87e..d1c79fb8c6 100644 --- a/py/vtproto/mysqlctl_pb2.py +++ b/py/vtproto/mysqlctl_pb2.py @@ -18,6 +18,7 @@ _sym_db = _symbol_database.Default() DESCRIPTOR = _descriptor.FileDescriptor( name='mysqlctl.proto', package='mysqlctl', + syntax='proto3', serialized_pb=_b('\n\x0emysqlctl.proto\x12\x08mysqlctl\"\x0e\n\x0cStartRequest\"\x0f\n\rStartResponse\"*\n\x0fShutdownRequest\x12\x17\n\x0fwait_for_mysqld\x18\x01 \x01(\x08\"\x12\n\x10ShutdownResponse\"\x18\n\x16RunMysqlUpgradeRequest\"\x19\n\x17RunMysqlUpgradeResponse2\xe5\x01\n\x08MysqlCtl\x12:\n\x05Start\x12\x16.mysqlctl.StartRequest\x1a\x17.mysqlctl.StartResponse\"\x00\x12\x43\n\x08Shutdown\x12\x19.mysqlctl.ShutdownRequest\x1a\x1a.mysqlctl.ShutdownResponse\"\x00\x12X\n\x0fRunMysqlUpgrade\x12 .mysqlctl.RunMysqlUpgradeRequest\x1a!.mysqlctl.RunMysqlUpgradeResponse\"\x00\x62\x06proto3') ) _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -40,6 +41,7 @@ _STARTREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -63,6 +65,7 @@ _STARTRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -93,6 +96,7 @@ _SHUTDOWNREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -116,6 +120,7 @@ _SHUTDOWNRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -139,6 +144,7 @@ _RUNMYSQLUPGRADEREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -162,6 +168,7 @@ _RUNMYSQLUPGRADERESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -258,7 +265,7 @@ class EarlyAdopterMysqlCtlStub(object): def RunMysqlUpgrade(self, request): raise NotImplementedError() RunMysqlUpgrade.async = None -def early_adopter_create_MysqlCtl_server(servicer, port, root_certificates, key_chain_pairs): +def early_adopter_create_MysqlCtl_server(servicer, port, private_key=None, certificate_chain=None): import mysqlctl_pb2 import mysqlctl_pb2 import mysqlctl_pb2 @@ -282,8 +289,8 @@ def early_adopter_create_MysqlCtl_server(servicer, port, root_certificates, key_ mysqlctl_pb2.StartResponse.SerializeToString, ), } - return implementations.secure_server("mysqlctl.MysqlCtl", method_service_descriptions, port, root_certificates, key_chain_pairs) -def early_adopter_create_MysqlCtl_stub(host, port): + return implementations.server("mysqlctl.MysqlCtl", method_service_descriptions, port, private_key=private_key, certificate_chain=certificate_chain) +def early_adopter_create_MysqlCtl_stub(host, port, metadata_transformer=None, secure=False, root_certificates=None, private_key=None, certificate_chain=None, server_host_override=None): import mysqlctl_pb2 import mysqlctl_pb2 import mysqlctl_pb2 @@ -304,5 +311,5 @@ def early_adopter_create_MysqlCtl_stub(host, port): mysqlctl_pb2.StartResponse.FromString, ), } - return implementations.insecure_stub("mysqlctl.MysqlCtl", method_invocation_descriptions, host, port) + return implementations.stub("mysqlctl.MysqlCtl", method_invocation_descriptions, host, port, metadata_transformer=metadata_transformer, secure=secure, root_certificates=root_certificates, private_key=private_key, certificate_chain=certificate_chain, server_host_override=server_host_override) # @@protoc_insertion_point(module_scope) diff --git a/py/vtproto/query_pb2.py b/py/vtproto/query_pb2.py index 64fdaf517a..9af1f5be52 100644 --- a/py/vtproto/query_pb2.py +++ b/py/vtproto/query_pb2.py @@ -20,6 +20,7 @@ import vtrpc_pb2 as vtrpc__pb2 DESCRIPTOR = _descriptor.FileDescriptor( name='query.proto', package='query', + syntax='proto3', serialized_pb=_b('\n\x0bquery.proto\x12\x05query\x1a\x0etopodata.proto\x1a\x0bvtrpc.proto\"T\n\x06Target\x12\x10\n\x08keyspace\x18\x01 \x01(\t\x12\r\n\x05shard\x18\x02 \x01(\t\x12)\n\x0btablet_type\x18\x03 \x01(\x0e\x32\x14.topodata.TabletType\"\"\n\x0eVTGateCallerID\x12\x10\n\x08username\x18\x01 \x01(\t\"\x92\x03\n\x0c\x42indVariable\x12&\n\x04type\x18\x01 \x01(\x0e\x32\x18.query.BindVariable.Type\x12\x13\n\x0bvalue_bytes\x18\x02 \x01(\x0c\x12\x11\n\tvalue_int\x18\x03 \x01(\x03\x12\x12\n\nvalue_uint\x18\x04 \x01(\x04\x12\x13\n\x0bvalue_float\x18\x05 \x01(\x01\x12\x18\n\x10value_bytes_list\x18\x06 \x03(\x0c\x12\x16\n\x0evalue_int_list\x18\x07 \x03(\x03\x12\x17\n\x0fvalue_uint_list\x18\x08 \x03(\x04\x12\x18\n\x10value_float_list\x18\t \x03(\x01\"\xa3\x01\n\x04Type\x12\r\n\tTYPE_NULL\x10\x00\x12\x0e\n\nTYPE_BYTES\x10\x01\x12\x0c\n\x08TYPE_INT\x10\x02\x12\r\n\tTYPE_UINT\x10\x03\x12\x0e\n\nTYPE_FLOAT\x10\x04\x12\x13\n\x0fTYPE_BYTES_LIST\x10\x05\x12\x11\n\rTYPE_INT_LIST\x10\x06\x12\x12\n\x0eTYPE_UINT_LIST\x10\x07\x12\x13\n\x0fTYPE_FLOAT_LIST\x10\x08\"\xa2\x01\n\nBoundQuery\x12\x0b\n\x03sql\x18\x01 \x01(\x0c\x12<\n\x0e\x62ind_variables\x18\x02 \x03(\x0b\x32$.query.BoundQuery.BindVariablesEntry\x1aI\n\x12\x42indVariablesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\"\n\x05value\x18\x02 \x01(\x0b\x32\x13.query.BindVariable:\x02\x38\x01\"\xa1\x07\n\x05\x46ield\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x1f\n\x04type\x18\x02 \x01(\x0e\x32\x11.query.Field.Type\x12\r\n\x05\x66lags\x18\x03 \x01(\x03\"\xe1\x03\n\x04Type\x12\x10\n\x0cTYPE_DECIMAL\x10\x00\x12\r\n\tTYPE_TINY\x10\x01\x12\x0e\n\nTYPE_SHORT\x10\x02\x12\r\n\tTYPE_LONG\x10\x03\x12\x0e\n\nTYPE_FLOAT\x10\x04\x12\x0f\n\x0bTYPE_DOUBLE\x10\x05\x12\r\n\tTYPE_NULL\x10\x06\x12\x12\n\x0eTYPE_TIMESTAMP\x10\x07\x12\x11\n\rTYPE_LONGLONG\x10\x08\x12\x0e\n\nTYPE_INT24\x10\t\x12\r\n\tTYPE_DATE\x10\n\x12\r\n\tTYPE_TIME\x10\x0b\x12\x11\n\rTYPE_DATETIME\x10\x0c\x12\r\n\tTYPE_YEAR\x10\r\x12\x10\n\x0cTYPE_NEWDATE\x10\x0e\x12\x10\n\x0cTYPE_VARCHAR\x10\x0f\x12\x0c\n\x08TYPE_BIT\x10\x10\x12\x14\n\x0fTYPE_NEWDECIMAL\x10\xf6\x01\x12\x0e\n\tTYPE_ENUM\x10\xf7\x01\x12\r\n\x08TYPE_SET\x10\xf8\x01\x12\x13\n\x0eTYPE_TINY_BLOB\x10\xf9\x01\x12\x15\n\x10TYPE_MEDIUM_BLOB\x10\xfa\x01\x12\x13\n\x0eTYPE_LONG_BLOB\x10\xfb\x01\x12\x0e\n\tTYPE_BLOB\x10\xfc\x01\x12\x14\n\x0fTYPE_VAR_STRING\x10\xfd\x01\x12\x10\n\x0bTYPE_STRING\x10\xfe\x01\x12\x12\n\rTYPE_GEOMETRY\x10\xff\x01\"\xf5\x02\n\x04\x46lag\x12\x15\n\x11VT_ZEROVALUE_FLAG\x10\x00\x12\x14\n\x10VT_NOT_NULL_FLAG\x10\x01\x12\x13\n\x0fVT_PRI_KEY_FLAG\x10\x02\x12\x16\n\x12VT_UNIQUE_KEY_FLAG\x10\x04\x12\x18\n\x14VT_MULTIPLE_KEY_FLAG\x10\x08\x12\x10\n\x0cVT_BLOB_FLAG\x10\x10\x12\x14\n\x10VT_UNSIGNED_FLAG\x10 \x12\x14\n\x10VT_ZEROFILL_FLAG\x10@\x12\x13\n\x0eVT_BINARY_FLAG\x10\x80\x01\x12\x11\n\x0cVT_ENUM_FLAG\x10\x80\x02\x12\x1b\n\x16VT_AUTO_INCREMENT_FLAG\x10\x80\x04\x12\x16\n\x11VT_TIMESTAMP_FLAG\x10\x80\x08\x12\x10\n\x0bVT_SET_FLAG\x10\x80\x10\x12\x1d\n\x18VT_NO_DEFAULT_VALUE_FLAG\x10\x80 \x12\x1a\n\x15VT_ON_UPDATE_NOW_FLAG\x10\x80@\x12\x11\n\x0bVT_NUM_FLAG\x10\x80\x80\x02\"\x15\n\x03Row\x12\x0e\n\x06values\x18\x01 \x03(\x0c\"o\n\x0bQueryResult\x12\x1c\n\x06\x66ields\x18\x01 \x03(\x0b\x32\x0c.query.Field\x12\x15\n\rrows_affected\x18\x02 \x01(\x04\x12\x11\n\tinsert_id\x18\x03 \x01(\x04\x12\x18\n\x04rows\x18\x04 \x03(\x0b\x32\n.query.Row\"\x98\x01\n\x13GetSessionIdRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x10\n\x08keyspace\x18\x03 \x01(\t\x12\r\n\x05shard\x18\x04 \x01(\t\"*\n\x14GetSessionIdResponse\x12\x12\n\nsession_id\x18\x01 \x01(\x03\"\xdf\x01\n\x0e\x45xecuteRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12 \n\x05query\x18\x04 \x01(\x0b\x32\x11.query.BoundQuery\x12\x16\n\x0etransaction_id\x18\x05 \x01(\x03\x12\x12\n\nsession_id\x18\x06 \x01(\x03\"5\n\x0f\x45xecuteResponse\x12\"\n\x06result\x18\x01 \x01(\x0b\x32\x12.query.QueryResult\"\xfe\x01\n\x13\x45xecuteBatchRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12\"\n\x07queries\x18\x04 \x03(\x0b\x32\x11.query.BoundQuery\x12\x16\n\x0e\x61s_transaction\x18\x05 \x01(\x08\x12\x16\n\x0etransaction_id\x18\x06 \x01(\x03\x12\x12\n\nsession_id\x18\x07 \x01(\x03\";\n\x14\x45xecuteBatchResponse\x12#\n\x07results\x18\x01 \x03(\x0b\x32\x12.query.QueryResult\"\xcd\x01\n\x14StreamExecuteRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12 \n\x05query\x18\x04 \x01(\x0b\x32\x11.query.BoundQuery\x12\x12\n\nsession_id\x18\x05 \x01(\x03\";\n\x15StreamExecuteResponse\x12\"\n\x06result\x18\x01 \x01(\x0b\x32\x12.query.QueryResult\"\xa3\x01\n\x0c\x42\x65ginRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12\x12\n\nsession_id\x18\x04 \x01(\x03\"\'\n\rBeginResponse\x12\x16\n\x0etransaction_id\x18\x01 \x01(\x03\"\xbc\x01\n\rCommitRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12\x16\n\x0etransaction_id\x18\x04 \x01(\x03\x12\x12\n\nsession_id\x18\x05 \x01(\x03\"\x10\n\x0e\x43ommitResponse\"\xbe\x01\n\x0fRollbackRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12\x16\n\x0etransaction_id\x18\x04 \x01(\x03\x12\x12\n\nsession_id\x18\x05 \x01(\x03\"\x12\n\x10RollbackResponse\"\xf5\x01\n\x11SplitQueryRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12 \n\x05query\x18\x04 \x01(\x0b\x32\x11.query.BoundQuery\x12\x14\n\x0csplit_column\x18\x05 \x01(\t\x12\x13\n\x0bsplit_count\x18\x06 \x01(\x03\x12\x12\n\nsession_id\x18\x07 \x01(\x03\"A\n\nQuerySplit\x12 \n\x05query\x18\x01 \x01(\x0b\x32\x11.query.BoundQuery\x12\x11\n\trow_count\x18\x02 \x01(\x03\"8\n\x12SplitQueryResponse\x12\"\n\x07queries\x18\x01 \x03(\x0b\x32\x11.query.QuerySplit\"\x15\n\x13StreamHealthRequest\"W\n\rRealtimeStats\x12\x14\n\x0chealth_error\x18\x01 \x01(\t\x12\x1d\n\x15seconds_behind_master\x18\x02 \x01(\r\x12\x11\n\tcpu_usage\x18\x03 \x01(\x01\"\x93\x01\n\x14StreamHealthResponse\x12\x1d\n\x06target\x18\x01 \x01(\x0b\x32\r.query.Target\x12.\n&tablet_externally_reparented_timestamp\x18\x02 \x01(\x03\x12,\n\x0erealtime_stats\x18\x03 \x01(\x0b\x32\x14.query.RealtimeStatsB\x1a\n\x18\x63om.youtube.vitess.protob\x06proto3') , dependencies=[topodata__pb2.DESCRIPTOR,vtrpc__pb2.DESCRIPTOR,]) @@ -314,6 +315,7 @@ _TARGET = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -344,6 +346,7 @@ _VTGATECALLERID = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -431,6 +434,7 @@ _BINDVARIABLE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -468,6 +472,7 @@ _BOUNDQUERY_BINDVARIABLESENTRY = _descriptor.Descriptor( ], options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')), is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -504,6 +509,7 @@ _BOUNDQUERY = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -550,6 +556,7 @@ _FIELD = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -580,6 +587,7 @@ _ROW = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -631,6 +639,7 @@ _QUERYRESULT = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -682,6 +691,7 @@ _GETSESSIONIDREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -712,6 +722,7 @@ _GETSESSIONIDRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -777,6 +788,7 @@ _EXECUTEREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -807,6 +819,7 @@ _EXECUTERESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -879,6 +892,7 @@ _EXECUTEBATCHREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -909,6 +923,7 @@ _EXECUTEBATCHRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -967,6 +982,7 @@ _STREAMEXECUTEREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -997,6 +1013,7 @@ _STREAMEXECUTERESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1048,6 +1065,7 @@ _BEGINREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1078,6 +1096,7 @@ _BEGINRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1136,6 +1155,7 @@ _COMMITREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1159,6 +1179,7 @@ _COMMITRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1217,6 +1238,7 @@ _ROLLBACKREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1240,6 +1262,7 @@ _ROLLBACKRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1312,6 +1335,7 @@ _SPLITQUERYREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1349,6 +1373,7 @@ _QUERYSPLIT = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1379,6 +1404,7 @@ _SPLITQUERYRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1402,6 +1428,7 @@ _STREAMHEALTHREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1446,6 +1473,7 @@ _REALTIMESTATS = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1490,6 +1518,7 @@ _STREAMHEALTHRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], diff --git a/py/vtproto/queryservice_pb2.py b/py/vtproto/queryservice_pb2.py index 07a1463525..6d68de0a5f 100644 --- a/py/vtproto/queryservice_pb2.py +++ b/py/vtproto/queryservice_pb2.py @@ -19,6 +19,7 @@ import query_pb2 as query__pb2 DESCRIPTOR = _descriptor.FileDescriptor( name='queryservice.proto', package='queryservice', + syntax='proto3', serialized_pb=_b('\n\x12queryservice.proto\x12\x0cqueryservice\x1a\x0bquery.proto2\xe9\x04\n\x05Query\x12I\n\x0cGetSessionId\x12\x1a.query.GetSessionIdRequest\x1a\x1b.query.GetSessionIdResponse\"\x00\x12:\n\x07\x45xecute\x12\x15.query.ExecuteRequest\x1a\x16.query.ExecuteResponse\"\x00\x12I\n\x0c\x45xecuteBatch\x12\x1a.query.ExecuteBatchRequest\x1a\x1b.query.ExecuteBatchResponse\"\x00\x12N\n\rStreamExecute\x12\x1b.query.StreamExecuteRequest\x1a\x1c.query.StreamExecuteResponse\"\x00\x30\x01\x12\x34\n\x05\x42\x65gin\x12\x13.query.BeginRequest\x1a\x14.query.BeginResponse\"\x00\x12\x37\n\x06\x43ommit\x12\x14.query.CommitRequest\x1a\x15.query.CommitResponse\"\x00\x12=\n\x08Rollback\x12\x16.query.RollbackRequest\x1a\x17.query.RollbackResponse\"\x00\x12\x43\n\nSplitQuery\x12\x18.query.SplitQueryRequest\x1a\x19.query.SplitQueryResponse\"\x00\x12K\n\x0cStreamHealth\x12\x1a.query.StreamHealthRequest\x1a\x1b.query.StreamHealthResponse\"\x00\x30\x01\x62\x06proto3') , dependencies=[query__pb2.DESCRIPTOR,]) @@ -109,7 +110,7 @@ class EarlyAdopterQueryStub(object): def StreamHealth(self, request): raise NotImplementedError() StreamHealth.async = None -def early_adopter_create_Query_server(servicer, port, root_certificates, key_chain_pairs): +def early_adopter_create_Query_server(servicer, port, private_key=None, certificate_chain=None): import query_pb2 import query_pb2 import query_pb2 @@ -175,8 +176,8 @@ def early_adopter_create_Query_server(servicer, port, root_certificates, key_cha query_pb2.StreamHealthResponse.SerializeToString, ), } - return implementations.secure_server("queryservice.Query", method_service_descriptions, port, root_certificates, key_chain_pairs) -def early_adopter_create_Query_stub(host, port): + return implementations.server("queryservice.Query", method_service_descriptions, port, private_key=private_key, certificate_chain=certificate_chain) +def early_adopter_create_Query_stub(host, port, metadata_transformer=None, secure=False, root_certificates=None, private_key=None, certificate_chain=None, server_host_override=None): import query_pb2 import query_pb2 import query_pb2 @@ -233,5 +234,5 @@ def early_adopter_create_Query_stub(host, port): query_pb2.StreamHealthResponse.FromString, ), } - return implementations.insecure_stub("queryservice.Query", method_invocation_descriptions, host, port) + return implementations.stub("queryservice.Query", method_invocation_descriptions, host, port, metadata_transformer=metadata_transformer, secure=secure, root_certificates=root_certificates, private_key=private_key, certificate_chain=certificate_chain, server_host_override=server_host_override) # @@protoc_insertion_point(module_scope) diff --git a/py/vtproto/replicationdata_pb2.py b/py/vtproto/replicationdata_pb2.py index e5a7defb8a..bf22a17774 100644 --- a/py/vtproto/replicationdata_pb2.py +++ b/py/vtproto/replicationdata_pb2.py @@ -18,6 +18,7 @@ _sym_db = _symbol_database.Default() DESCRIPTOR = _descriptor.FileDescriptor( name='replicationdata.proto', package='replicationdata', + syntax='proto3', serialized_pb=_b('\n\x15replicationdata.proto\x12\x0freplicationdata\"\xb6\x01\n\x06Status\x12\x10\n\x08position\x18\x01 \x01(\t\x12\x18\n\x10slave_io_running\x18\x02 \x01(\x08\x12\x19\n\x11slave_sql_running\x18\x03 \x01(\x08\x12\x1d\n\x15seconds_behind_master\x18\x04 \x01(\r\x12\x13\n\x0bmaster_host\x18\x05 \x01(\t\x12\x13\n\x0bmaster_port\x18\x06 \x01(\x05\x12\x1c\n\x14master_connect_retry\x18\x07 \x01(\x05\x62\x06proto3') ) _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -89,6 +90,7 @@ _STATUS = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], diff --git a/py/vtproto/tableacl_pb2.py b/py/vtproto/tableacl_pb2.py index adf0a9853e..a727ee1d8e 100644 --- a/py/vtproto/tableacl_pb2.py +++ b/py/vtproto/tableacl_pb2.py @@ -18,6 +18,7 @@ _sym_db = _symbol_database.Default() DESCRIPTOR = _descriptor.FileDescriptor( name='tableacl.proto', package='tableacl', + syntax='proto3', serialized_pb=_b('\n\x0etableacl.proto\x12\x08tableacl\"q\n\x0eTableGroupSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x1f\n\x17table_names_or_prefixes\x18\x02 \x03(\t\x12\x0f\n\x07readers\x18\x03 \x03(\t\x12\x0f\n\x07writers\x18\x04 \x03(\t\x12\x0e\n\x06\x61\x64mins\x18\x05 \x03(\t\"8\n\x06\x43onfig\x12.\n\x0ctable_groups\x18\x01 \x03(\x0b\x32\x18.tableacl.TableGroupSpecb\x06proto3') ) _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -75,6 +76,7 @@ _TABLEGROUPSPEC = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -105,6 +107,7 @@ _CONFIG = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], diff --git a/py/vtproto/tabletmanagerdata_pb2.py b/py/vtproto/tabletmanagerdata_pb2.py index 34e0cdf563..699115c879 100644 --- a/py/vtproto/tabletmanagerdata_pb2.py +++ b/py/vtproto/tabletmanagerdata_pb2.py @@ -22,6 +22,7 @@ import logutil_pb2 as logutil__pb2 DESCRIPTOR = _descriptor.FileDescriptor( name='tabletmanagerdata.proto', package='tabletmanagerdata', + syntax='proto3', serialized_pb=_b('\n\x17tabletmanagerdata.proto\x12\x11tabletmanagerdata\x1a\x0bquery.proto\x1a\x0etopodata.proto\x1a\x15replicationdata.proto\x1a\rlogutil.proto\"\x93\x01\n\x0fTableDefinition\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06schema\x18\x02 \x01(\t\x12\x0f\n\x07\x63olumns\x18\x03 \x03(\t\x12\x1b\n\x13primary_key_columns\x18\x04 \x03(\t\x12\x0c\n\x04type\x18\x05 \x01(\t\x12\x13\n\x0b\x64\x61ta_length\x18\x06 \x01(\x04\x12\x11\n\trow_count\x18\x07 \x01(\x04\"{\n\x10SchemaDefinition\x12\x17\n\x0f\x64\x61tabase_schema\x18\x01 \x01(\t\x12=\n\x11table_definitions\x18\x02 \x03(\x0b\x32\".tabletmanagerdata.TableDefinition\x12\x0f\n\x07version\x18\x03 \x01(\t\"\xc1\x01\n\x0eUserPermission\x12\x0c\n\x04host\x18\x01 \x01(\t\x12\x0c\n\x04user\x18\x02 \x01(\t\x12\x19\n\x11password_checksum\x18\x03 \x01(\x04\x12\x45\n\nprivileges\x18\x04 \x03(\x0b\x32\x31.tabletmanagerdata.UserPermission.PrivilegesEntry\x1a\x31\n\x0fPrivilegesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xae\x01\n\x0c\x44\x62Permission\x12\x0c\n\x04host\x18\x01 \x01(\t\x12\n\n\x02\x64\x62\x18\x02 \x01(\t\x12\x0c\n\x04user\x18\x03 \x01(\t\x12\x43\n\nprivileges\x18\x04 \x03(\x0b\x32/.tabletmanagerdata.DbPermission.PrivilegesEntry\x1a\x31\n\x0fPrivilegesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xa4\x01\n\x0eHostPermission\x12\x0c\n\x04host\x18\x01 \x01(\t\x12\n\n\x02\x64\x62\x18\x02 \x01(\t\x12\x45\n\nprivileges\x18\x03 \x03(\x0b\x32\x31.tabletmanagerdata.HostPermission.PrivilegesEntry\x1a\x31\n\x0fPrivilegesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xc0\x01\n\x0bPermissions\x12;\n\x10user_permissions\x18\x01 \x03(\x0b\x32!.tabletmanagerdata.UserPermission\x12\x37\n\x0e\x64\x62_permissions\x18\x02 \x03(\x0b\x32\x1f.tabletmanagerdata.DbPermission\x12;\n\x10host_permissions\x18\x03 \x03(\x0b\x32!.tabletmanagerdata.HostPermission\",\n\x0b\x42lpPosition\x12\x0b\n\x03uid\x18\x01 \x01(\r\x12\x10\n\x08position\x18\x02 \x01(\t\"\x1e\n\x0bPingRequest\x12\x0f\n\x07payload\x18\x01 \x01(\t\"\x1f\n\x0cPingResponse\x12\x0f\n\x07payload\x18\x01 \x01(\t\" \n\x0cSleepRequest\x12\x10\n\x08\x64uration\x18\x01 \x01(\x03\"\x0f\n\rSleepResponse\"\xaf\x01\n\x12\x45xecuteHookRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x12\n\nparameters\x18\x02 \x03(\t\x12\x46\n\textra_env\x18\x03 \x03(\x0b\x32\x33.tabletmanagerdata.ExecuteHookRequest.ExtraEnvEntry\x1a/\n\rExtraEnvEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"J\n\x13\x45xecuteHookResponse\x12\x13\n\x0b\x65xit_status\x18\x01 \x01(\x03\x12\x0e\n\x06stdout\x18\x02 \x01(\t\x12\x0e\n\x06stderr\x18\x03 \x01(\t\"Q\n\x10GetSchemaRequest\x12\x0e\n\x06tables\x18\x01 \x03(\t\x12\x15\n\rinclude_views\x18\x02 \x01(\x08\x12\x16\n\x0e\x65xclude_tables\x18\x03 \x03(\t\"S\n\x11GetSchemaResponse\x12>\n\x11schema_definition\x18\x01 \x01(\x0b\x32#.tabletmanagerdata.SchemaDefinition\"\x17\n\x15GetPermissionsRequest\"M\n\x16GetPermissionsResponse\x12\x33\n\x0bpermissions\x18\x01 \x01(\x0b\x32\x1e.tabletmanagerdata.Permissions\"\x14\n\x12SetReadOnlyRequest\"\x15\n\x13SetReadOnlyResponse\"\x15\n\x13SetReadWriteRequest\"\x16\n\x14SetReadWriteResponse\">\n\x11\x43hangeTypeRequest\x12)\n\x0btablet_type\x18\x01 \x01(\x0e\x32\x14.topodata.TabletType\"\x14\n\x12\x43hangeTypeResponse\"\x0e\n\x0cScrapRequest\"\x0f\n\rScrapResponse\"\x15\n\x13RefreshStateRequest\"\x16\n\x14RefreshStateResponse\"B\n\x15RunHealthCheckRequest\x12)\n\x0btablet_type\x18\x01 \x01(\x0e\x32\x14.topodata.TabletType\"\x18\n\x16RunHealthCheckResponse\"\x15\n\x13ReloadSchemaRequest\"\x16\n\x14ReloadSchemaResponse\"(\n\x16PreflightSchemaRequest\x12\x0e\n\x06\x63hange\x18\x01 \x01(\t\"\x90\x01\n\x17PreflightSchemaResponse\x12:\n\rbefore_schema\x18\x01 \x01(\x0b\x32#.tabletmanagerdata.SchemaDefinition\x12\x39\n\x0c\x61\x66ter_schema\x18\x02 \x01(\x0b\x32#.tabletmanagerdata.SchemaDefinition\"\xc2\x01\n\x12\x41pplySchemaRequest\x12\x0b\n\x03sql\x18\x01 \x01(\t\x12\r\n\x05\x66orce\x18\x02 \x01(\x08\x12\x19\n\x11\x61llow_replication\x18\x03 \x01(\x08\x12:\n\rbefore_schema\x18\x04 \x01(\x0b\x32#.tabletmanagerdata.SchemaDefinition\x12\x39\n\x0c\x61\x66ter_schema\x18\x05 \x01(\x0b\x32#.tabletmanagerdata.SchemaDefinition\"\x8c\x01\n\x13\x41pplySchemaResponse\x12:\n\rbefore_schema\x18\x01 \x01(\x0b\x32#.tabletmanagerdata.SchemaDefinition\x12\x39\n\x0c\x61\x66ter_schema\x18\x02 \x01(\x0b\x32#.tabletmanagerdata.SchemaDefinition\"\x91\x01\n\x18\x45xecuteFetchAsDbaRequest\x12\r\n\x05query\x18\x01 \x01(\t\x12\x0f\n\x07\x64\x62_name\x18\x02 \x01(\t\x12\x10\n\x08max_rows\x18\x03 \x01(\x04\x12\x13\n\x0bwant_fields\x18\x04 \x01(\x08\x12\x17\n\x0f\x64isable_binlogs\x18\x05 \x01(\x08\x12\x15\n\rreload_schema\x18\x06 \x01(\x08\"?\n\x19\x45xecuteFetchAsDbaResponse\x12\"\n\x06result\x18\x01 \x01(\x0b\x32\x12.query.QueryResult\"P\n\x18\x45xecuteFetchAsAppRequest\x12\r\n\x05query\x18\x01 \x01(\t\x12\x10\n\x08max_rows\x18\x02 \x01(\x04\x12\x13\n\x0bwant_fields\x18\x03 \x01(\x08\"?\n\x19\x45xecuteFetchAsAppResponse\x12\"\n\x06result\x18\x01 \x01(\x0b\x32\x12.query.QueryResult\"\x14\n\x12SlaveStatusRequest\">\n\x13SlaveStatusResponse\x12\'\n\x06status\x18\x01 \x01(\x0b\x32\x17.replicationdata.Status\"\x17\n\x15MasterPositionRequest\"*\n\x16MasterPositionResponse\x12\x10\n\x08position\x18\x01 \x01(\t\"\x12\n\x10StopSlaveRequest\"\x13\n\x11StopSlaveResponse\"A\n\x17StopSlaveMinimumRequest\x12\x10\n\x08position\x18\x01 \x01(\t\x12\x14\n\x0cwait_timeout\x18\x02 \x01(\x03\",\n\x18StopSlaveMinimumResponse\x12\x10\n\x08position\x18\x01 \x01(\t\"\x13\n\x11StartSlaveRequest\"\x14\n\x12StartSlaveResponse\"8\n!TabletExternallyReparentedRequest\x12\x13\n\x0b\x65xternal_id\x18\x01 \x01(\t\"$\n\"TabletExternallyReparentedResponse\" \n\x1eTabletExternallyElectedRequest\"!\n\x1fTabletExternallyElectedResponse\"\x12\n\x10GetSlavesRequest\"\"\n\x11GetSlavesResponse\x12\r\n\x05\x61\x64\x64rs\x18\x01 \x03(\t\"d\n\x16WaitBlpPositionRequest\x12\x34\n\x0c\x62lp_position\x18\x01 \x01(\x0b\x32\x1e.tabletmanagerdata.BlpPosition\x12\x14\n\x0cwait_timeout\x18\x02 \x01(\x03\"\x19\n\x17WaitBlpPositionResponse\"\x10\n\x0eStopBlpRequest\"H\n\x0fStopBlpResponse\x12\x35\n\rblp_positions\x18\x01 \x03(\x0b\x32\x1e.tabletmanagerdata.BlpPosition\"\x11\n\x0fStartBlpRequest\"\x12\n\x10StartBlpResponse\"a\n\x12RunBlpUntilRequest\x12\x35\n\rblp_positions\x18\x01 \x03(\x0b\x32\x1e.tabletmanagerdata.BlpPosition\x12\x14\n\x0cwait_timeout\x18\x02 \x01(\x03\"\'\n\x13RunBlpUntilResponse\x12\x10\n\x08position\x18\x01 \x01(\t\"\x19\n\x17ResetReplicationRequest\"\x1a\n\x18ResetReplicationResponse\"\x13\n\x11InitMasterRequest\"&\n\x12InitMasterResponse\x12\x10\n\x08position\x18\x01 \x01(\t\"\x99\x01\n\x1ePopulateReparentJournalRequest\x12\x17\n\x0ftime_created_ns\x18\x01 \x01(\x03\x12\x13\n\x0b\x61\x63tion_name\x18\x02 \x01(\t\x12+\n\x0cmaster_alias\x18\x03 \x01(\x0b\x32\x15.topodata.TabletAlias\x12\x1c\n\x14replication_position\x18\x04 \x01(\t\"!\n\x1fPopulateReparentJournalResponse\"p\n\x10InitSlaveRequest\x12%\n\x06parent\x18\x01 \x01(\x0b\x32\x15.topodata.TabletAlias\x12\x1c\n\x14replication_position\x18\x02 \x01(\t\x12\x17\n\x0ftime_created_ns\x18\x03 \x01(\x03\"\x13\n\x11InitSlaveResponse\"\x15\n\x13\x44\x65moteMasterRequest\"(\n\x14\x44\x65moteMasterResponse\x12\x10\n\x08position\x18\x01 \x01(\t\"3\n\x1fPromoteSlaveWhenCaughtUpRequest\x12\x10\n\x08position\x18\x01 \x01(\t\"4\n PromoteSlaveWhenCaughtUpResponse\x12\x10\n\x08position\x18\x01 \x01(\t\"\x19\n\x17SlaveWasPromotedRequest\"\x1a\n\x18SlaveWasPromotedResponse\"m\n\x10SetMasterRequest\x12%\n\x06parent\x18\x01 \x01(\x0b\x32\x15.topodata.TabletAlias\x12\x17\n\x0ftime_created_ns\x18\x02 \x01(\x03\x12\x19\n\x11\x66orce_start_slave\x18\x03 \x01(\x08\"\x13\n\x11SetMasterResponse\"A\n\x18SlaveWasRestartedRequest\x12%\n\x06parent\x18\x01 \x01(\x0b\x32\x15.topodata.TabletAlias\"\x1b\n\x19SlaveWasRestartedResponse\"$\n\"StopReplicationAndGetStatusRequest\"N\n#StopReplicationAndGetStatusResponse\x12\'\n\x06status\x18\x01 \x01(\x0b\x32\x17.replicationdata.Status\"\x15\n\x13PromoteSlaveRequest\"(\n\x14PromoteSlaveResponse\x12\x10\n\x08position\x18\x01 \x01(\t\"$\n\rBackupRequest\x12\x13\n\x0b\x63oncurrency\x18\x01 \x01(\x03\"/\n\x0e\x42\x61\x63kupResponse\x12\x1d\n\x05\x65vent\x18\x01 \x01(\x0b\x32\x0e.logutil.Eventb\x06proto3') , dependencies=[query__pb2.DESCRIPTOR,topodata__pb2.DESCRIPTOR,replicationdata__pb2.DESCRIPTOR,logutil__pb2.DESCRIPTOR,]) @@ -94,6 +95,7 @@ _TABLEDEFINITION = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -138,6 +140,7 @@ _SCHEMADEFINITION = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -175,6 +178,7 @@ _USERPERMISSION_PRIVILEGESENTRY = _descriptor.Descriptor( ], options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')), is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -225,6 +229,7 @@ _USERPERMISSION = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -262,6 +267,7 @@ _DBPERMISSION_PRIVILEGESENTRY = _descriptor.Descriptor( ], options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')), is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -312,6 +318,7 @@ _DBPERMISSION = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -349,6 +356,7 @@ _HOSTPERMISSION_PRIVILEGESENTRY = _descriptor.Descriptor( ], options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')), is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -392,6 +400,7 @@ _HOSTPERMISSION = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -436,6 +445,7 @@ _PERMISSIONS = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -473,6 +483,7 @@ _BLPPOSITION = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -503,6 +514,7 @@ _PINGREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -533,6 +545,7 @@ _PINGRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -563,6 +576,7 @@ _SLEEPREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -586,6 +600,7 @@ _SLEEPRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -623,6 +638,7 @@ _EXECUTEHOOKREQUEST_EXTRAENVENTRY = _descriptor.Descriptor( ], options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')), is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -666,6 +682,7 @@ _EXECUTEHOOKREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -710,6 +727,7 @@ _EXECUTEHOOKRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -754,6 +772,7 @@ _GETSCHEMAREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -784,6 +803,7 @@ _GETSCHEMARESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -807,6 +827,7 @@ _GETPERMISSIONSREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -837,6 +858,7 @@ _GETPERMISSIONSRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -860,6 +882,7 @@ _SETREADONLYREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -883,6 +906,7 @@ _SETREADONLYRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -906,6 +930,7 @@ _SETREADWRITEREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -929,6 +954,7 @@ _SETREADWRITERESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -959,6 +985,7 @@ _CHANGETYPEREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -982,6 +1009,7 @@ _CHANGETYPERESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1005,6 +1033,7 @@ _SCRAPREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1028,6 +1057,7 @@ _SCRAPRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1051,6 +1081,7 @@ _REFRESHSTATEREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1074,6 +1105,7 @@ _REFRESHSTATERESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1104,6 +1136,7 @@ _RUNHEALTHCHECKREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1127,6 +1160,7 @@ _RUNHEALTHCHECKRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1150,6 +1184,7 @@ _RELOADSCHEMAREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1173,6 +1208,7 @@ _RELOADSCHEMARESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1203,6 +1239,7 @@ _PREFLIGHTSCHEMAREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1240,6 +1277,7 @@ _PREFLIGHTSCHEMARESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1298,6 +1336,7 @@ _APPLYSCHEMAREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1335,6 +1374,7 @@ _APPLYSCHEMARESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1400,6 +1440,7 @@ _EXECUTEFETCHASDBAREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1430,6 +1471,7 @@ _EXECUTEFETCHASDBARESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1474,6 +1516,7 @@ _EXECUTEFETCHASAPPREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1504,6 +1547,7 @@ _EXECUTEFETCHASAPPRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1527,6 +1571,7 @@ _SLAVESTATUSREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1557,6 +1602,7 @@ _SLAVESTATUSRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1580,6 +1626,7 @@ _MASTERPOSITIONREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1610,6 +1657,7 @@ _MASTERPOSITIONRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1633,6 +1681,7 @@ _STOPSLAVEREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1656,6 +1705,7 @@ _STOPSLAVERESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1693,6 +1743,7 @@ _STOPSLAVEMINIMUMREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1723,6 +1774,7 @@ _STOPSLAVEMINIMUMRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1746,6 +1798,7 @@ _STARTSLAVEREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1769,6 +1822,7 @@ _STARTSLAVERESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1799,6 +1853,7 @@ _TABLETEXTERNALLYREPARENTEDREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1822,6 +1877,7 @@ _TABLETEXTERNALLYREPARENTEDRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1845,6 +1901,7 @@ _TABLETEXTERNALLYELECTEDREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1868,6 +1925,7 @@ _TABLETEXTERNALLYELECTEDRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1891,6 +1949,7 @@ _GETSLAVESREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1921,6 +1980,7 @@ _GETSLAVESRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1958,6 +2018,7 @@ _WAITBLPPOSITIONREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1981,6 +2042,7 @@ _WAITBLPPOSITIONRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -2004,6 +2066,7 @@ _STOPBLPREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -2034,6 +2097,7 @@ _STOPBLPRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -2057,6 +2121,7 @@ _STARTBLPREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -2080,6 +2145,7 @@ _STARTBLPRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -2117,6 +2183,7 @@ _RUNBLPUNTILREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -2147,6 +2214,7 @@ _RUNBLPUNTILRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -2170,6 +2238,7 @@ _RESETREPLICATIONREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -2193,6 +2262,7 @@ _RESETREPLICATIONRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -2216,6 +2286,7 @@ _INITMASTERREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -2246,6 +2317,7 @@ _INITMASTERRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -2297,6 +2369,7 @@ _POPULATEREPARENTJOURNALREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -2320,6 +2393,7 @@ _POPULATEREPARENTJOURNALRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -2364,6 +2438,7 @@ _INITSLAVEREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -2387,6 +2462,7 @@ _INITSLAVERESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -2410,6 +2486,7 @@ _DEMOTEMASTERREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -2440,6 +2517,7 @@ _DEMOTEMASTERRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -2470,6 +2548,7 @@ _PROMOTESLAVEWHENCAUGHTUPREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -2500,6 +2579,7 @@ _PROMOTESLAVEWHENCAUGHTUPRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -2523,6 +2603,7 @@ _SLAVEWASPROMOTEDREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -2546,6 +2627,7 @@ _SLAVEWASPROMOTEDRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -2590,6 +2672,7 @@ _SETMASTERREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -2613,6 +2696,7 @@ _SETMASTERRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -2643,6 +2727,7 @@ _SLAVEWASRESTARTEDREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -2666,6 +2751,7 @@ _SLAVEWASRESTARTEDRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -2689,6 +2775,7 @@ _STOPREPLICATIONANDGETSTATUSREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -2719,6 +2806,7 @@ _STOPREPLICATIONANDGETSTATUSRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -2742,6 +2830,7 @@ _PROMOTESLAVEREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -2772,6 +2861,7 @@ _PROMOTESLAVERESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -2802,6 +2892,7 @@ _BACKUPREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -2832,6 +2923,7 @@ _BACKUPRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], diff --git a/py/vtproto/tabletmanagerservice_pb2.py b/py/vtproto/tabletmanagerservice_pb2.py index dde789f8a1..30679cfd2f 100644 --- a/py/vtproto/tabletmanagerservice_pb2.py +++ b/py/vtproto/tabletmanagerservice_pb2.py @@ -19,6 +19,7 @@ import tabletmanagerdata_pb2 as tabletmanagerdata__pb2 DESCRIPTOR = _descriptor.FileDescriptor( name='tabletmanagerservice.proto', package='tabletmanagerservice', + syntax='proto3', serialized_pb=_b('\n\x1atabletmanagerservice.proto\x12\x14tabletmanagerservice\x1a\x17tabletmanagerdata.proto2\x83 \n\rTabletManager\x12I\n\x04Ping\x12\x1e.tabletmanagerdata.PingRequest\x1a\x1f.tabletmanagerdata.PingResponse\"\x00\x12L\n\x05Sleep\x12\x1f.tabletmanagerdata.SleepRequest\x1a .tabletmanagerdata.SleepResponse\"\x00\x12^\n\x0b\x45xecuteHook\x12%.tabletmanagerdata.ExecuteHookRequest\x1a&.tabletmanagerdata.ExecuteHookResponse\"\x00\x12X\n\tGetSchema\x12#.tabletmanagerdata.GetSchemaRequest\x1a$.tabletmanagerdata.GetSchemaResponse\"\x00\x12g\n\x0eGetPermissions\x12(.tabletmanagerdata.GetPermissionsRequest\x1a).tabletmanagerdata.GetPermissionsResponse\"\x00\x12^\n\x0bSetReadOnly\x12%.tabletmanagerdata.SetReadOnlyRequest\x1a&.tabletmanagerdata.SetReadOnlyResponse\"\x00\x12\x61\n\x0cSetReadWrite\x12&.tabletmanagerdata.SetReadWriteRequest\x1a\'.tabletmanagerdata.SetReadWriteResponse\"\x00\x12[\n\nChangeType\x12$.tabletmanagerdata.ChangeTypeRequest\x1a%.tabletmanagerdata.ChangeTypeResponse\"\x00\x12L\n\x05Scrap\x12\x1f.tabletmanagerdata.ScrapRequest\x1a .tabletmanagerdata.ScrapResponse\"\x00\x12\x61\n\x0cRefreshState\x12&.tabletmanagerdata.RefreshStateRequest\x1a\'.tabletmanagerdata.RefreshStateResponse\"\x00\x12g\n\x0eRunHealthCheck\x12(.tabletmanagerdata.RunHealthCheckRequest\x1a).tabletmanagerdata.RunHealthCheckResponse\"\x00\x12\x61\n\x0cReloadSchema\x12&.tabletmanagerdata.ReloadSchemaRequest\x1a\'.tabletmanagerdata.ReloadSchemaResponse\"\x00\x12j\n\x0fPreflightSchema\x12).tabletmanagerdata.PreflightSchemaRequest\x1a*.tabletmanagerdata.PreflightSchemaResponse\"\x00\x12^\n\x0b\x41pplySchema\x12%.tabletmanagerdata.ApplySchemaRequest\x1a&.tabletmanagerdata.ApplySchemaResponse\"\x00\x12p\n\x11\x45xecuteFetchAsDba\x12+.tabletmanagerdata.ExecuteFetchAsDbaRequest\x1a,.tabletmanagerdata.ExecuteFetchAsDbaResponse\"\x00\x12p\n\x11\x45xecuteFetchAsApp\x12+.tabletmanagerdata.ExecuteFetchAsAppRequest\x1a,.tabletmanagerdata.ExecuteFetchAsAppResponse\"\x00\x12^\n\x0bSlaveStatus\x12%.tabletmanagerdata.SlaveStatusRequest\x1a&.tabletmanagerdata.SlaveStatusResponse\"\x00\x12g\n\x0eMasterPosition\x12(.tabletmanagerdata.MasterPositionRequest\x1a).tabletmanagerdata.MasterPositionResponse\"\x00\x12X\n\tStopSlave\x12#.tabletmanagerdata.StopSlaveRequest\x1a$.tabletmanagerdata.StopSlaveResponse\"\x00\x12m\n\x10StopSlaveMinimum\x12*.tabletmanagerdata.StopSlaveMinimumRequest\x1a+.tabletmanagerdata.StopSlaveMinimumResponse\"\x00\x12[\n\nStartSlave\x12$.tabletmanagerdata.StartSlaveRequest\x1a%.tabletmanagerdata.StartSlaveResponse\"\x00\x12\x8b\x01\n\x1aTabletExternallyReparented\x12\x34.tabletmanagerdata.TabletExternallyReparentedRequest\x1a\x35.tabletmanagerdata.TabletExternallyReparentedResponse\"\x00\x12\x82\x01\n\x17TabletExternallyElected\x12\x31.tabletmanagerdata.TabletExternallyElectedRequest\x1a\x32.tabletmanagerdata.TabletExternallyElectedResponse\"\x00\x12X\n\tGetSlaves\x12#.tabletmanagerdata.GetSlavesRequest\x1a$.tabletmanagerdata.GetSlavesResponse\"\x00\x12j\n\x0fWaitBlpPosition\x12).tabletmanagerdata.WaitBlpPositionRequest\x1a*.tabletmanagerdata.WaitBlpPositionResponse\"\x00\x12R\n\x07StopBlp\x12!.tabletmanagerdata.StopBlpRequest\x1a\".tabletmanagerdata.StopBlpResponse\"\x00\x12U\n\x08StartBlp\x12\".tabletmanagerdata.StartBlpRequest\x1a#.tabletmanagerdata.StartBlpResponse\"\x00\x12^\n\x0bRunBlpUntil\x12%.tabletmanagerdata.RunBlpUntilRequest\x1a&.tabletmanagerdata.RunBlpUntilResponse\"\x00\x12m\n\x10ResetReplication\x12*.tabletmanagerdata.ResetReplicationRequest\x1a+.tabletmanagerdata.ResetReplicationResponse\"\x00\x12[\n\nInitMaster\x12$.tabletmanagerdata.InitMasterRequest\x1a%.tabletmanagerdata.InitMasterResponse\"\x00\x12\x82\x01\n\x17PopulateReparentJournal\x12\x31.tabletmanagerdata.PopulateReparentJournalRequest\x1a\x32.tabletmanagerdata.PopulateReparentJournalResponse\"\x00\x12X\n\tInitSlave\x12#.tabletmanagerdata.InitSlaveRequest\x1a$.tabletmanagerdata.InitSlaveResponse\"\x00\x12\x61\n\x0c\x44\x65moteMaster\x12&.tabletmanagerdata.DemoteMasterRequest\x1a\'.tabletmanagerdata.DemoteMasterResponse\"\x00\x12\x85\x01\n\x18PromoteSlaveWhenCaughtUp\x12\x32.tabletmanagerdata.PromoteSlaveWhenCaughtUpRequest\x1a\x33.tabletmanagerdata.PromoteSlaveWhenCaughtUpResponse\"\x00\x12m\n\x10SlaveWasPromoted\x12*.tabletmanagerdata.SlaveWasPromotedRequest\x1a+.tabletmanagerdata.SlaveWasPromotedResponse\"\x00\x12X\n\tSetMaster\x12#.tabletmanagerdata.SetMasterRequest\x1a$.tabletmanagerdata.SetMasterResponse\"\x00\x12p\n\x11SlaveWasRestarted\x12+.tabletmanagerdata.SlaveWasRestartedRequest\x1a,.tabletmanagerdata.SlaveWasRestartedResponse\"\x00\x12\x8e\x01\n\x1bStopReplicationAndGetStatus\x12\x35.tabletmanagerdata.StopReplicationAndGetStatusRequest\x1a\x36.tabletmanagerdata.StopReplicationAndGetStatusResponse\"\x00\x12\x61\n\x0cPromoteSlave\x12&.tabletmanagerdata.PromoteSlaveRequest\x1a\'.tabletmanagerdata.PromoteSlaveResponse\"\x00\x12Q\n\x06\x42\x61\x63kup\x12 .tabletmanagerdata.BackupRequest\x1a!.tabletmanagerdata.BackupResponse\"\x00\x30\x01\x62\x06proto3') , dependencies=[tabletmanagerdata__pb2.DESCRIPTOR,]) @@ -326,7 +327,7 @@ class EarlyAdopterTabletManagerStub(object): def Backup(self, request): raise NotImplementedError() Backup.async = None -def early_adopter_create_TabletManager_server(servicer, port, root_certificates, key_chain_pairs): +def early_adopter_create_TabletManager_server(servicer, port, private_key=None, certificate_chain=None): import tabletmanagerdata_pb2 import tabletmanagerdata_pb2 import tabletmanagerdata_pb2 @@ -609,8 +610,8 @@ def early_adopter_create_TabletManager_server(servicer, port, root_certificates, tabletmanagerdata_pb2.WaitBlpPositionResponse.SerializeToString, ), } - return implementations.secure_server("tabletmanagerservice.TabletManager", method_service_descriptions, port, root_certificates, key_chain_pairs) -def early_adopter_create_TabletManager_stub(host, port): + return implementations.server("tabletmanagerservice.TabletManager", method_service_descriptions, port, private_key=private_key, certificate_chain=certificate_chain) +def early_adopter_create_TabletManager_stub(host, port, metadata_transformer=None, secure=False, root_certificates=None, private_key=None, certificate_chain=None, server_host_override=None): import tabletmanagerdata_pb2 import tabletmanagerdata_pb2 import tabletmanagerdata_pb2 @@ -853,5 +854,5 @@ def early_adopter_create_TabletManager_stub(host, port): tabletmanagerdata_pb2.WaitBlpPositionResponse.FromString, ), } - return implementations.insecure_stub("tabletmanagerservice.TabletManager", method_invocation_descriptions, host, port) + return implementations.stub("tabletmanagerservice.TabletManager", method_invocation_descriptions, host, port, metadata_transformer=metadata_transformer, secure=secure, root_certificates=root_certificates, private_key=private_key, certificate_chain=certificate_chain, server_host_override=server_host_override) # @@protoc_insertion_point(module_scope) diff --git a/py/vtproto/topodata_pb2.py b/py/vtproto/topodata_pb2.py index c903daae2b..13f5993487 100644 --- a/py/vtproto/topodata_pb2.py +++ b/py/vtproto/topodata_pb2.py @@ -19,6 +19,7 @@ _sym_db = _symbol_database.Default() DESCRIPTOR = _descriptor.FileDescriptor( name='topodata.proto', package='topodata', + syntax='proto3', serialized_pb=_b('\n\x0etopodata.proto\x12\x08topodata\"&\n\x08KeyRange\x12\r\n\x05start\x18\x01 \x01(\x0c\x12\x0b\n\x03\x65nd\x18\x02 \x01(\x0c\"(\n\x0bTabletAlias\x12\x0c\n\x04\x63\x65ll\x18\x01 \x01(\t\x12\x0b\n\x03uid\x18\x02 \x01(\r\"\xf1\x03\n\x06Tablet\x12$\n\x05\x61lias\x18\x01 \x01(\x0b\x32\x15.topodata.TabletAlias\x12\x10\n\x08hostname\x18\x02 \x01(\t\x12\n\n\x02ip\x18\x03 \x01(\t\x12/\n\x08port_map\x18\x04 \x03(\x0b\x32\x1d.topodata.Tablet.PortMapEntry\x12\x10\n\x08keyspace\x18\x05 \x01(\t\x12\r\n\x05shard\x18\x06 \x01(\t\x12%\n\tkey_range\x18\x07 \x01(\x0b\x32\x12.topodata.KeyRange\x12\"\n\x04type\x18\x08 \x01(\x0e\x32\x14.topodata.TabletType\x12\x18\n\x10\x64\x62_name_override\x18\t \x01(\t\x12(\n\x04tags\x18\n \x03(\x0b\x32\x1a.topodata.Tablet.TagsEntry\x12\x33\n\nhealth_map\x18\x0b \x03(\x0b\x32\x1f.topodata.Tablet.HealthMapEntry\x1a.\n\x0cPortMapEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x30\n\x0eHealthMapEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xcb\x04\n\x05Shard\x12+\n\x0cmaster_alias\x18\x01 \x01(\x0b\x32\x15.topodata.TabletAlias\x12%\n\tkey_range\x18\x02 \x01(\x0b\x32\x12.topodata.KeyRange\x12\x30\n\x0cserved_types\x18\x03 \x03(\x0b\x32\x1a.topodata.Shard.ServedType\x12\x32\n\rsource_shards\x18\x04 \x03(\x0b\x32\x1b.topodata.Shard.SourceShard\x12\r\n\x05\x63\x65lls\x18\x05 \x03(\t\x12\x36\n\x0ftablet_controls\x18\x06 \x03(\x0b\x32\x1d.topodata.Shard.TabletControl\x1a\x46\n\nServedType\x12)\n\x0btablet_type\x18\x01 \x01(\x0e\x32\x14.topodata.TabletType\x12\r\n\x05\x63\x65lls\x18\x02 \x03(\t\x1ar\n\x0bSourceShard\x12\x0b\n\x03uid\x18\x01 \x01(\r\x12\x10\n\x08keyspace\x18\x02 \x01(\t\x12\r\n\x05shard\x18\x03 \x01(\t\x12%\n\tkey_range\x18\x04 \x01(\x0b\x32\x12.topodata.KeyRange\x12\x0e\n\x06tables\x18\x05 \x03(\t\x1a\x84\x01\n\rTabletControl\x12)\n\x0btablet_type\x18\x01 \x01(\x0e\x32\x14.topodata.TabletType\x12\r\n\x05\x63\x65lls\x18\x02 \x03(\t\x12\x1d\n\x15\x64isable_query_service\x18\x03 \x01(\x08\x12\x1a\n\x12\x62lacklisted_tables\x18\x04 \x03(\t\"\x8a\x02\n\x08Keyspace\x12\x1c\n\x14sharding_column_name\x18\x01 \x01(\t\x12\x36\n\x14sharding_column_type\x18\x02 \x01(\x0e\x32\x18.topodata.KeyspaceIdType\x12\x19\n\x11split_shard_count\x18\x03 \x01(\x05\x12\x33\n\x0cserved_froms\x18\x04 \x03(\x0b\x32\x1d.topodata.Keyspace.ServedFrom\x1aX\n\nServedFrom\x12)\n\x0btablet_type\x18\x01 \x01(\x0e\x32\x14.topodata.TabletType\x12\r\n\x05\x63\x65lls\x18\x02 \x03(\t\x12\x10\n\x08keyspace\x18\x03 \x01(\t\"w\n\x10ShardReplication\x12.\n\x05nodes\x18\x01 \x03(\x0b\x32\x1f.topodata.ShardReplication.Node\x1a\x33\n\x04Node\x12+\n\x0ctablet_alias\x18\x01 \x01(\x0b\x32\x15.topodata.TabletAlias\"\xf1\x01\n\x08\x45ndPoint\x12\x0b\n\x03uid\x18\x01 \x01(\r\x12\x0c\n\x04host\x18\x02 \x01(\t\x12\x31\n\x08port_map\x18\x03 \x03(\x0b\x32\x1f.topodata.EndPoint.PortMapEntry\x12\x35\n\nhealth_map\x18\x04 \x03(\x0b\x32!.topodata.EndPoint.HealthMapEntry\x1a.\n\x0cPortMapEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\x1a\x30\n\x0eHealthMapEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"0\n\tEndPoints\x12#\n\x07\x65ntries\x18\x01 \x03(\x0b\x32\x12.topodata.EndPoint\"T\n\x08SrvShard\x12\x0c\n\x04name\x18\x01 \x01(\t\x12%\n\tkey_range\x18\x02 \x01(\x0b\x32\x12.topodata.KeyRange\x12\x13\n\x0bmaster_cell\x18\x03 \x01(\t\"E\n\x0eShardReference\x12\x0c\n\x04name\x18\x01 \x01(\t\x12%\n\tkey_range\x18\x02 \x01(\x0b\x32\x12.topodata.KeyRange\"\xb1\x03\n\x0bSrvKeyspace\x12;\n\npartitions\x18\x01 \x03(\x0b\x32\'.topodata.SrvKeyspace.KeyspacePartition\x12\x1c\n\x14sharding_column_name\x18\x02 \x01(\t\x12\x36\n\x14sharding_column_type\x18\x03 \x01(\x0e\x32\x18.topodata.KeyspaceIdType\x12\x35\n\x0bserved_from\x18\x04 \x03(\x0b\x32 .topodata.SrvKeyspace.ServedFrom\x12\x19\n\x11split_shard_count\x18\x05 \x01(\x05\x1ar\n\x11KeyspacePartition\x12)\n\x0bserved_type\x18\x01 \x01(\x0e\x32\x14.topodata.TabletType\x12\x32\n\x10shard_references\x18\x02 \x03(\x0b\x32\x18.topodata.ShardReference\x1aI\n\nServedFrom\x12)\n\x0btablet_type\x18\x01 \x01(\x0e\x32\x14.topodata.TabletType\x12\x10\n\x08keyspace\x18\x02 \x01(\t*2\n\x0eKeyspaceIdType\x12\t\n\x05UNSET\x10\x00\x12\n\n\x06UINT64\x10\x01\x12\t\n\x05\x42YTES\x10\x02*\xb8\x01\n\nTabletType\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x08\n\x04IDLE\x10\x01\x12\n\n\x06MASTER\x10\x02\x12\x0b\n\x07REPLICA\x10\x03\x12\n\n\x06RDONLY\x10\x04\x12\t\n\x05\x42\x41TCH\x10\x04\x12\t\n\x05SPARE\x10\x05\x12\x10\n\x0c\x45XPERIMENTAL\x10\x06\x12\x12\n\x0eSCHEMA_UPGRADE\x10\x07\x12\n\n\x06\x42\x41\x43KUP\x10\x08\x12\x0b\n\x07RESTORE\x10\t\x12\n\n\x06WORKER\x10\n\x12\t\n\x05SCRAP\x10\x0b\x1a\x02\x10\x01\x42\x1a\n\x18\x63om.youtube.vitess.protob\x06proto3') ) _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -165,6 +166,7 @@ _KEYRANGE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -202,6 +204,7 @@ _TABLETALIAS = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -239,6 +242,7 @@ _TABLET_PORTMAPENTRY = _descriptor.Descriptor( ], options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')), is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -275,6 +279,7 @@ _TABLET_TAGSENTRY = _descriptor.Descriptor( ], options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')), is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -311,6 +316,7 @@ _TABLET_HEALTHMAPENTRY = _descriptor.Descriptor( ], options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')), is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -410,6 +416,7 @@ _TABLET = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -447,6 +454,7 @@ _SHARD_SERVEDTYPE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -504,6 +512,7 @@ _SHARD_SOURCESHARD = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -554,6 +563,7 @@ _SHARD_TABLETCONTROL = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -618,6 +628,7 @@ _SHARD = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -662,6 +673,7 @@ _KEYSPACE_SERVEDFROM = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -712,6 +724,7 @@ _KEYSPACE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -742,6 +755,7 @@ _SHARDREPLICATION_NODE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -771,6 +785,7 @@ _SHARDREPLICATION = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -808,6 +823,7 @@ _ENDPOINT_PORTMAPENTRY = _descriptor.Descriptor( ], options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')), is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -844,6 +860,7 @@ _ENDPOINT_HEALTHMAPENTRY = _descriptor.Descriptor( ], options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')), is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -894,6 +911,7 @@ _ENDPOINT = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -924,6 +942,7 @@ _ENDPOINTS = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -968,6 +987,7 @@ _SRVSHARD = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1005,6 +1025,7 @@ _SHARDREFERENCE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1042,6 +1063,7 @@ _SRVKEYSPACE_KEYSPACEPARTITION = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1078,6 +1100,7 @@ _SRVKEYSPACE_SERVEDFROM = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1135,6 +1158,7 @@ _SRVKEYSPACE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], diff --git a/py/vtproto/vtctldata_pb2.py b/py/vtproto/vtctldata_pb2.py index 6898c533b0..c7699711b5 100644 --- a/py/vtproto/vtctldata_pb2.py +++ b/py/vtproto/vtctldata_pb2.py @@ -19,6 +19,7 @@ import logutil_pb2 as logutil__pb2 DESCRIPTOR = _descriptor.FileDescriptor( name='vtctldata.proto', package='vtctldata', + syntax='proto3', serialized_pb=_b('\n\x0fvtctldata.proto\x12\tvtctldata\x1a\rlogutil.proto\"X\n\x1a\x45xecuteVtctlCommandRequest\x12\x0c\n\x04\x61rgs\x18\x01 \x03(\t\x12\x16\n\x0e\x61\x63tion_timeout\x18\x02 \x01(\x03\x12\x14\n\x0clock_timeout\x18\x03 \x01(\x03\"<\n\x1b\x45xecuteVtctlCommandResponse\x12\x1d\n\x05\x65vent\x18\x01 \x01(\x0b\x32\x0e.logutil.Eventb\x06proto3') , dependencies=[logutil__pb2.DESCRIPTOR,]) @@ -63,6 +64,7 @@ _EXECUTEVTCTLCOMMANDREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -93,6 +95,7 @@ _EXECUTEVTCTLCOMMANDRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], diff --git a/py/vtproto/vtctlservice_pb2.py b/py/vtproto/vtctlservice_pb2.py index b181993251..e464e7a21d 100644 --- a/py/vtproto/vtctlservice_pb2.py +++ b/py/vtproto/vtctlservice_pb2.py @@ -19,6 +19,7 @@ import vtctldata_pb2 as vtctldata__pb2 DESCRIPTOR = _descriptor.FileDescriptor( name='vtctlservice.proto', package='vtctlservice', + syntax='proto3', serialized_pb=_b('\n\x12vtctlservice.proto\x12\x0cvtctlservice\x1a\x0fvtctldata.proto2q\n\x05Vtctl\x12h\n\x13\x45xecuteVtctlCommand\x12%.vtctldata.ExecuteVtctlCommandRequest\x1a&.vtctldata.ExecuteVtctlCommandResponse\"\x00\x30\x01\x62\x06proto3') , dependencies=[vtctldata__pb2.DESCRIPTOR,]) @@ -53,7 +54,7 @@ class EarlyAdopterVtctlStub(object): def ExecuteVtctlCommand(self, request): raise NotImplementedError() ExecuteVtctlCommand.async = None -def early_adopter_create_Vtctl_server(servicer, port, root_certificates, key_chain_pairs): +def early_adopter_create_Vtctl_server(servicer, port, private_key=None, certificate_chain=None): import vtctldata_pb2 import vtctldata_pb2 method_service_descriptions = { @@ -63,8 +64,8 @@ def early_adopter_create_Vtctl_server(servicer, port, root_certificates, key_cha vtctldata_pb2.ExecuteVtctlCommandResponse.SerializeToString, ), } - return implementations.secure_server("vtctlservice.Vtctl", method_service_descriptions, port, root_certificates, key_chain_pairs) -def early_adopter_create_Vtctl_stub(host, port): + return implementations.server("vtctlservice.Vtctl", method_service_descriptions, port, private_key=private_key, certificate_chain=certificate_chain) +def early_adopter_create_Vtctl_stub(host, port, metadata_transformer=None, secure=False, root_certificates=None, private_key=None, certificate_chain=None, server_host_override=None): import vtctldata_pb2 import vtctldata_pb2 method_invocation_descriptions = { @@ -73,5 +74,5 @@ def early_adopter_create_Vtctl_stub(host, port): vtctldata_pb2.ExecuteVtctlCommandResponse.FromString, ), } - return implementations.insecure_stub("vtctlservice.Vtctl", method_invocation_descriptions, host, port) + return implementations.stub("vtctlservice.Vtctl", method_invocation_descriptions, host, port, metadata_transformer=metadata_transformer, secure=secure, root_certificates=root_certificates, private_key=private_key, certificate_chain=certificate_chain, server_host_override=server_host_override) # @@protoc_insertion_point(module_scope) diff --git a/py/vtproto/vtgate_pb2.py b/py/vtproto/vtgate_pb2.py index 0c43e82a37..995bd46c28 100644 --- a/py/vtproto/vtgate_pb2.py +++ b/py/vtproto/vtgate_pb2.py @@ -21,6 +21,7 @@ import vtrpc_pb2 as vtrpc__pb2 DESCRIPTOR = _descriptor.FileDescriptor( name='vtgate.proto', package='vtgate', + syntax='proto3', serialized_pb=_b('\n\x0cvtgate.proto\x12\x06vtgate\x1a\x0bquery.proto\x1a\x0etopodata.proto\x1a\x0bvtrpc.proto\"\x9e\x01\n\x07Session\x12\x16\n\x0ein_transaction\x18\x01 \x01(\x08\x12\x34\n\x0eshard_sessions\x18\x02 \x03(\x0b\x32\x1c.vtgate.Session.ShardSession\x1a\x45\n\x0cShardSession\x12\x1d\n\x06target\x18\x01 \x01(\x0b\x32\r.query.Target\x12\x16\n\x0etransaction_id\x18\x02 \x01(\x03\"\xbf\x01\n\x0e\x45xecuteRequest\x12\"\n\tcaller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12 \n\x07session\x18\x02 \x01(\x0b\x32\x0f.vtgate.Session\x12 \n\x05query\x18\x03 \x01(\x0b\x32\x11.query.BoundQuery\x12)\n\x0btablet_type\x18\x04 \x01(\x0e\x32\x14.topodata.TabletType\x12\x1a\n\x12not_in_transaction\x18\x05 \x01(\x08\"w\n\x0f\x45xecuteResponse\x12\x1e\n\x05\x65rror\x18\x01 \x01(\x0b\x32\x0f.vtrpc.RPCError\x12 \n\x07session\x18\x02 \x01(\x0b\x32\x0f.vtgate.Session\x12\"\n\x06result\x18\x03 \x01(\x0b\x32\x12.query.QueryResult\"\xe7\x01\n\x14\x45xecuteShardsRequest\x12\"\n\tcaller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12 \n\x07session\x18\x02 \x01(\x0b\x32\x0f.vtgate.Session\x12 \n\x05query\x18\x03 \x01(\x0b\x32\x11.query.BoundQuery\x12\x10\n\x08keyspace\x18\x04 \x01(\t\x12\x0e\n\x06shards\x18\x05 \x03(\t\x12)\n\x0btablet_type\x18\x06 \x01(\x0e\x32\x14.topodata.TabletType\x12\x1a\n\x12not_in_transaction\x18\x07 \x01(\x08\"}\n\x15\x45xecuteShardsResponse\x12\x1e\n\x05\x65rror\x18\x01 \x01(\x0b\x32\x0f.vtrpc.RPCError\x12 \n\x07session\x18\x02 \x01(\x0b\x32\x0f.vtgate.Session\x12\"\n\x06result\x18\x03 \x01(\x0b\x32\x12.query.QueryResult\"\xf2\x01\n\x19\x45xecuteKeyspaceIdsRequest\x12\"\n\tcaller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12 \n\x07session\x18\x02 \x01(\x0b\x32\x0f.vtgate.Session\x12 \n\x05query\x18\x03 \x01(\x0b\x32\x11.query.BoundQuery\x12\x10\n\x08keyspace\x18\x04 \x01(\t\x12\x14\n\x0ckeyspace_ids\x18\x05 \x03(\x0c\x12)\n\x0btablet_type\x18\x06 \x01(\x0e\x32\x14.topodata.TabletType\x12\x1a\n\x12not_in_transaction\x18\x07 \x01(\x08\"\x82\x01\n\x1a\x45xecuteKeyspaceIdsResponse\x12\x1e\n\x05\x65rror\x18\x01 \x01(\x0b\x32\x0f.vtrpc.RPCError\x12 \n\x07session\x18\x02 \x01(\x0b\x32\x0f.vtgate.Session\x12\"\n\x06result\x18\x03 \x01(\x0b\x32\x12.query.QueryResult\"\x82\x02\n\x17\x45xecuteKeyRangesRequest\x12\"\n\tcaller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12 \n\x07session\x18\x02 \x01(\x0b\x32\x0f.vtgate.Session\x12 \n\x05query\x18\x03 \x01(\x0b\x32\x11.query.BoundQuery\x12\x10\n\x08keyspace\x18\x04 \x01(\t\x12&\n\nkey_ranges\x18\x05 \x03(\x0b\x32\x12.topodata.KeyRange\x12)\n\x0btablet_type\x18\x06 \x01(\x0e\x32\x14.topodata.TabletType\x12\x1a\n\x12not_in_transaction\x18\x07 \x01(\x08\"\x80\x01\n\x18\x45xecuteKeyRangesResponse\x12\x1e\n\x05\x65rror\x18\x01 \x01(\x0b\x32\x0f.vtrpc.RPCError\x12 \n\x07session\x18\x02 \x01(\x0b\x32\x0f.vtgate.Session\x12\"\n\x06result\x18\x03 \x01(\x0b\x32\x12.query.QueryResult\"\xbd\x04\n\x17\x45xecuteEntityIdsRequest\x12\"\n\tcaller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12 \n\x07session\x18\x02 \x01(\x0b\x32\x0f.vtgate.Session\x12 \n\x05query\x18\x03 \x01(\x0b\x32\x11.query.BoundQuery\x12\x10\n\x08keyspace\x18\x04 \x01(\t\x12\x1a\n\x12\x65ntity_column_name\x18\x05 \x01(\t\x12\x45\n\x13\x65ntity_keyspace_ids\x18\x06 \x03(\x0b\x32(.vtgate.ExecuteEntityIdsRequest.EntityId\x12)\n\x0btablet_type\x18\x07 \x01(\x0e\x32\x14.topodata.TabletType\x12\x1a\n\x12not_in_transaction\x18\x08 \x01(\x08\x1a\xfd\x01\n\x08\x45ntityId\x12?\n\x08xid_type\x18\x01 \x01(\x0e\x32-.vtgate.ExecuteEntityIdsRequest.EntityId.Type\x12\x11\n\txid_bytes\x18\x02 \x01(\x0c\x12\x0f\n\x07xid_int\x18\x03 \x01(\x03\x12\x10\n\x08xid_uint\x18\x04 \x01(\x04\x12\x11\n\txid_float\x18\x05 \x01(\x01\x12\x13\n\x0bkeyspace_id\x18\x06 \x01(\x0c\"R\n\x04Type\x12\r\n\tTYPE_NULL\x10\x00\x12\x0e\n\nTYPE_BYTES\x10\x01\x12\x0c\n\x08TYPE_INT\x10\x02\x12\r\n\tTYPE_UINT\x10\x03\x12\x0e\n\nTYPE_FLOAT\x10\x04\"\x80\x01\n\x18\x45xecuteEntityIdsResponse\x12\x1e\n\x05\x65rror\x18\x01 \x01(\x0b\x32\x0f.vtrpc.RPCError\x12 \n\x07session\x18\x02 \x01(\x0b\x32\x0f.vtgate.Session\x12\"\n\x06result\x18\x03 \x01(\x0b\x32\x12.query.QueryResult\"U\n\x0f\x42oundShardQuery\x12 \n\x05query\x18\x01 \x01(\x0b\x32\x11.query.BoundQuery\x12\x10\n\x08keyspace\x18\x02 \x01(\t\x12\x0e\n\x06shards\x18\x03 \x03(\t\"\xce\x01\n\x19\x45xecuteBatchShardsRequest\x12\"\n\tcaller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12 \n\x07session\x18\x02 \x01(\x0b\x32\x0f.vtgate.Session\x12(\n\x07queries\x18\x03 \x03(\x0b\x32\x17.vtgate.BoundShardQuery\x12)\n\x0btablet_type\x18\x04 \x01(\x0e\x32\x14.topodata.TabletType\x12\x16\n\x0e\x61s_transaction\x18\x05 \x01(\x08\"\x83\x01\n\x1a\x45xecuteBatchShardsResponse\x12\x1e\n\x05\x65rror\x18\x01 \x01(\x0b\x32\x0f.vtrpc.RPCError\x12 \n\x07session\x18\x02 \x01(\x0b\x32\x0f.vtgate.Session\x12#\n\x07results\x18\x03 \x03(\x0b\x32\x12.query.QueryResult\"`\n\x14\x42oundKeyspaceIdQuery\x12 \n\x05query\x18\x01 \x01(\x0b\x32\x11.query.BoundQuery\x12\x10\n\x08keyspace\x18\x02 \x01(\t\x12\x14\n\x0ckeyspace_ids\x18\x03 \x03(\x0c\"\xd8\x01\n\x1e\x45xecuteBatchKeyspaceIdsRequest\x12\"\n\tcaller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12 \n\x07session\x18\x02 \x01(\x0b\x32\x0f.vtgate.Session\x12-\n\x07queries\x18\x03 \x03(\x0b\x32\x1c.vtgate.BoundKeyspaceIdQuery\x12)\n\x0btablet_type\x18\x04 \x01(\x0e\x32\x14.topodata.TabletType\x12\x16\n\x0e\x61s_transaction\x18\x05 \x01(\x08\"\x88\x01\n\x1f\x45xecuteBatchKeyspaceIdsResponse\x12\x1e\n\x05\x65rror\x18\x01 \x01(\x0b\x32\x0f.vtrpc.RPCError\x12 \n\x07session\x18\x02 \x01(\x0b\x32\x0f.vtgate.Session\x12#\n\x07results\x18\x03 \x03(\x0b\x32\x12.query.QueryResult\"\x87\x01\n\x14StreamExecuteRequest\x12\"\n\tcaller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12 \n\x05query\x18\x02 \x01(\x0b\x32\x11.query.BoundQuery\x12)\n\x0btablet_type\x18\x03 \x01(\x0e\x32\x14.topodata.TabletType\"[\n\x15StreamExecuteResponse\x12\x1e\n\x05\x65rror\x18\x01 \x01(\x0b\x32\x0f.vtrpc.RPCError\x12\"\n\x06result\x18\x02 \x01(\x0b\x32\x12.query.QueryResult\"\xaf\x01\n\x1aStreamExecuteShardsRequest\x12\"\n\tcaller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12 \n\x05query\x18\x02 \x01(\x0b\x32\x11.query.BoundQuery\x12\x10\n\x08keyspace\x18\x03 \x01(\t\x12\x0e\n\x06shards\x18\x04 \x03(\t\x12)\n\x0btablet_type\x18\x05 \x01(\x0e\x32\x14.topodata.TabletType\"a\n\x1bStreamExecuteShardsResponse\x12\x1e\n\x05\x65rror\x18\x01 \x01(\x0b\x32\x0f.vtrpc.RPCError\x12\"\n\x06result\x18\x02 \x01(\x0b\x32\x12.query.QueryResult\"\xba\x01\n\x1fStreamExecuteKeyspaceIdsRequest\x12\"\n\tcaller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12 \n\x05query\x18\x02 \x01(\x0b\x32\x11.query.BoundQuery\x12\x10\n\x08keyspace\x18\x03 \x01(\t\x12\x14\n\x0ckeyspace_ids\x18\x04 \x03(\x0c\x12)\n\x0btablet_type\x18\x05 \x01(\x0e\x32\x14.topodata.TabletType\"f\n StreamExecuteKeyspaceIdsResponse\x12\x1e\n\x05\x65rror\x18\x01 \x01(\x0b\x32\x0f.vtrpc.RPCError\x12\"\n\x06result\x18\x02 \x01(\x0b\x32\x12.query.QueryResult\"\xca\x01\n\x1dStreamExecuteKeyRangesRequest\x12\"\n\tcaller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12 \n\x05query\x18\x02 \x01(\x0b\x32\x11.query.BoundQuery\x12\x10\n\x08keyspace\x18\x03 \x01(\t\x12&\n\nkey_ranges\x18\x04 \x03(\x0b\x32\x12.topodata.KeyRange\x12)\n\x0btablet_type\x18\x05 \x01(\x0e\x32\x14.topodata.TabletType\"d\n\x1eStreamExecuteKeyRangesResponse\x12\x1e\n\x05\x65rror\x18\x01 \x01(\x0b\x32\x0f.vtrpc.RPCError\x12\"\n\x06result\x18\x02 \x01(\x0b\x32\x12.query.QueryResult\"2\n\x0c\x42\x65ginRequest\x12\"\n\tcaller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\"Q\n\rBeginResponse\x12\x1e\n\x05\x65rror\x18\x01 \x01(\x0b\x32\x0f.vtrpc.RPCError\x12 \n\x07session\x18\x02 \x01(\x0b\x32\x0f.vtgate.Session\"U\n\rCommitRequest\x12\"\n\tcaller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12 \n\x07session\x18\x02 \x01(\x0b\x32\x0f.vtgate.Session\"0\n\x0e\x43ommitResponse\x12\x1e\n\x05\x65rror\x18\x01 \x01(\x0b\x32\x0f.vtrpc.RPCError\"W\n\x0fRollbackRequest\x12\"\n\tcaller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12 \n\x07session\x18\x02 \x01(\x0b\x32\x0f.vtgate.Session\"2\n\x10RollbackResponse\x12\x1e\n\x05\x65rror\x18\x01 \x01(\x0b\x32\x0f.vtrpc.RPCError\"\x96\x01\n\x11SplitQueryRequest\x12\"\n\tcaller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x10\n\x08keyspace\x18\x02 \x01(\t\x12 \n\x05query\x18\x03 \x01(\x0b\x32\x11.query.BoundQuery\x12\x14\n\x0csplit_column\x18\x04 \x01(\t\x12\x13\n\x0bsplit_count\x18\x05 \x01(\x03\"\xf2\x02\n\x12SplitQueryResponse\x12/\n\x06splits\x18\x01 \x03(\x0b\x32\x1f.vtgate.SplitQueryResponse.Part\x1aH\n\x0cKeyRangePart\x12\x10\n\x08keyspace\x18\x01 \x01(\t\x12&\n\nkey_ranges\x18\x02 \x03(\x0b\x32\x12.topodata.KeyRange\x1a-\n\tShardPart\x12\x10\n\x08keyspace\x18\x01 \x01(\t\x12\x0e\n\x06shards\x18\x02 \x03(\t\x1a\xb1\x01\n\x04Part\x12 \n\x05query\x18\x01 \x01(\x0b\x32\x11.query.BoundQuery\x12?\n\x0ekey_range_part\x18\x02 \x01(\x0b\x32\'.vtgate.SplitQueryResponse.KeyRangePart\x12\x38\n\nshard_part\x18\x03 \x01(\x0b\x32$.vtgate.SplitQueryResponse.ShardPart\x12\x0c\n\x04size\x18\x04 \x01(\x03\")\n\x15GetSrvKeyspaceRequest\x12\x10\n\x08keyspace\x18\x01 \x01(\t\"E\n\x16GetSrvKeyspaceResponse\x12+\n\x0csrv_keyspace\x18\x01 \x01(\x0b\x32\x15.topodata.SrvKeyspaceB\x1a\n\x18\x63om.youtube.vitess.protob\x06proto3') , dependencies=[query__pb2.DESCRIPTOR,topodata__pb2.DESCRIPTOR,vtrpc__pb2.DESCRIPTOR,]) @@ -92,6 +93,7 @@ _SESSION_SHARDSESSION = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -128,6 +130,7 @@ _SESSION = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -186,6 +189,7 @@ _EXECUTEREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -230,6 +234,7 @@ _EXECUTERESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -302,6 +307,7 @@ _EXECUTESHARDSREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -346,6 +352,7 @@ _EXECUTESHARDSRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -418,6 +425,7 @@ _EXECUTEKEYSPACEIDSREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -462,6 +470,7 @@ _EXECUTEKEYSPACEIDSRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -534,6 +543,7 @@ _EXECUTEKEYRANGESREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -578,6 +588,7 @@ _EXECUTEKEYRANGESRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -644,6 +655,7 @@ _EXECUTEENTITYIDSREQUEST_ENTITYID = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -722,6 +734,7 @@ _EXECUTEENTITYIDSREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -766,6 +779,7 @@ _EXECUTEENTITYIDSRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -810,6 +824,7 @@ _BOUNDSHARDQUERY = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -868,6 +883,7 @@ _EXECUTEBATCHSHARDSREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -912,6 +928,7 @@ _EXECUTEBATCHSHARDSRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -956,6 +973,7 @@ _BOUNDKEYSPACEIDQUERY = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1014,6 +1032,7 @@ _EXECUTEBATCHKEYSPACEIDSREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1058,6 +1077,7 @@ _EXECUTEBATCHKEYSPACEIDSRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1102,6 +1122,7 @@ _STREAMEXECUTEREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1139,6 +1160,7 @@ _STREAMEXECUTERESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1197,6 +1219,7 @@ _STREAMEXECUTESHARDSREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1234,6 +1257,7 @@ _STREAMEXECUTESHARDSRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1292,6 +1316,7 @@ _STREAMEXECUTEKEYSPACEIDSREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1329,6 +1354,7 @@ _STREAMEXECUTEKEYSPACEIDSRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1387,6 +1413,7 @@ _STREAMEXECUTEKEYRANGESREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1424,6 +1451,7 @@ _STREAMEXECUTEKEYRANGESRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1454,6 +1482,7 @@ _BEGINREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1491,6 +1520,7 @@ _BEGINRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1528,6 +1558,7 @@ _COMMITREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1558,6 +1589,7 @@ _COMMITRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1595,6 +1627,7 @@ _ROLLBACKREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1625,6 +1658,7 @@ _ROLLBACKRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1683,6 +1717,7 @@ _SPLITQUERYREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1720,6 +1755,7 @@ _SPLITQUERYRESPONSE_KEYRANGEPART = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1756,6 +1792,7 @@ _SPLITQUERYRESPONSE_SHARDPART = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1806,6 +1843,7 @@ _SPLITQUERYRESPONSE_PART = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1835,6 +1873,7 @@ _SPLITQUERYRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1865,6 +1904,7 @@ _GETSRVKEYSPACEREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -1895,6 +1935,7 @@ _GETSRVKEYSPACERESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], diff --git a/py/vtproto/vtgateservice_pb2.py b/py/vtproto/vtgateservice_pb2.py index cdc37fa86f..6ef2d5a5c4 100644 --- a/py/vtproto/vtgateservice_pb2.py +++ b/py/vtproto/vtgateservice_pb2.py @@ -19,6 +19,7 @@ import vtgate_pb2 as vtgate__pb2 DESCRIPTOR = _descriptor.FileDescriptor( name='vtgateservice.proto', package='vtgateservice', + syntax='proto3', serialized_pb=_b('\n\x13vtgateservice.proto\x12\rvtgateservice\x1a\x0cvtgate.proto2\xd8\n\n\x06Vitess\x12<\n\x07\x45xecute\x12\x16.vtgate.ExecuteRequest\x1a\x17.vtgate.ExecuteResponse\"\x00\x12N\n\rExecuteShards\x12\x1c.vtgate.ExecuteShardsRequest\x1a\x1d.vtgate.ExecuteShardsResponse\"\x00\x12]\n\x12\x45xecuteKeyspaceIds\x12!.vtgate.ExecuteKeyspaceIdsRequest\x1a\".vtgate.ExecuteKeyspaceIdsResponse\"\x00\x12W\n\x10\x45xecuteKeyRanges\x12\x1f.vtgate.ExecuteKeyRangesRequest\x1a .vtgate.ExecuteKeyRangesResponse\"\x00\x12W\n\x10\x45xecuteEntityIds\x12\x1f.vtgate.ExecuteEntityIdsRequest\x1a .vtgate.ExecuteEntityIdsResponse\"\x00\x12]\n\x12\x45xecuteBatchShards\x12!.vtgate.ExecuteBatchShardsRequest\x1a\".vtgate.ExecuteBatchShardsResponse\"\x00\x12l\n\x17\x45xecuteBatchKeyspaceIds\x12&.vtgate.ExecuteBatchKeyspaceIdsRequest\x1a\'.vtgate.ExecuteBatchKeyspaceIdsResponse\"\x00\x12P\n\rStreamExecute\x12\x1c.vtgate.StreamExecuteRequest\x1a\x1d.vtgate.StreamExecuteResponse\"\x00\x30\x01\x12\x62\n\x13StreamExecuteShards\x12\".vtgate.StreamExecuteShardsRequest\x1a#.vtgate.StreamExecuteShardsResponse\"\x00\x30\x01\x12q\n\x18StreamExecuteKeyspaceIds\x12\'.vtgate.StreamExecuteKeyspaceIdsRequest\x1a(.vtgate.StreamExecuteKeyspaceIdsResponse\"\x00\x30\x01\x12k\n\x16StreamExecuteKeyRanges\x12%.vtgate.StreamExecuteKeyRangesRequest\x1a&.vtgate.StreamExecuteKeyRangesResponse\"\x00\x30\x01\x12\x36\n\x05\x42\x65gin\x12\x14.vtgate.BeginRequest\x1a\x15.vtgate.BeginResponse\"\x00\x12\x39\n\x06\x43ommit\x12\x15.vtgate.CommitRequest\x1a\x16.vtgate.CommitResponse\"\x00\x12?\n\x08Rollback\x12\x17.vtgate.RollbackRequest\x1a\x18.vtgate.RollbackResponse\"\x00\x12\x45\n\nSplitQuery\x12\x19.vtgate.SplitQueryRequest\x1a\x1a.vtgate.SplitQueryResponse\"\x00\x12Q\n\x0eGetSrvKeyspace\x12\x1d.vtgate.GetSrvKeyspaceRequest\x1a\x1e.vtgate.GetSrvKeyspaceResponse\"\x00\x42\x1f\n\x1d\x63om.youtube.vitess.proto.grpcb\x06proto3') , dependencies=[vtgate__pb2.DESCRIPTOR,]) @@ -160,7 +161,7 @@ class EarlyAdopterVitessStub(object): def GetSrvKeyspace(self, request): raise NotImplementedError() GetSrvKeyspace.async = None -def early_adopter_create_Vitess_server(servicer, port, root_certificates, key_chain_pairs): +def early_adopter_create_Vitess_server(servicer, port, private_key=None, certificate_chain=None): import vtgate_pb2 import vtgate_pb2 import vtgate_pb2 @@ -275,8 +276,8 @@ def early_adopter_create_Vitess_server(servicer, port, root_certificates, key_ch vtgate_pb2.StreamExecuteShardsResponse.SerializeToString, ), } - return implementations.secure_server("vtgateservice.Vitess", method_service_descriptions, port, root_certificates, key_chain_pairs) -def early_adopter_create_Vitess_stub(host, port): + return implementations.server("vtgateservice.Vitess", method_service_descriptions, port, private_key=private_key, certificate_chain=certificate_chain) +def early_adopter_create_Vitess_stub(host, port, metadata_transformer=None, secure=False, root_certificates=None, private_key=None, certificate_chain=None, server_host_override=None): import vtgate_pb2 import vtgate_pb2 import vtgate_pb2 @@ -375,5 +376,5 @@ def early_adopter_create_Vitess_stub(host, port): vtgate_pb2.StreamExecuteShardsResponse.FromString, ), } - return implementations.insecure_stub("vtgateservice.Vitess", method_invocation_descriptions, host, port) + return implementations.stub("vtgateservice.Vitess", method_invocation_descriptions, host, port, metadata_transformer=metadata_transformer, secure=secure, root_certificates=root_certificates, private_key=private_key, certificate_chain=certificate_chain, server_host_override=server_host_override) # @@protoc_insertion_point(module_scope) diff --git a/py/vtproto/vtrpc_pb2.py b/py/vtproto/vtrpc_pb2.py index f3c7d79218..639ab37a38 100644 --- a/py/vtproto/vtrpc_pb2.py +++ b/py/vtproto/vtrpc_pb2.py @@ -19,6 +19,7 @@ _sym_db = _symbol_database.Default() DESCRIPTOR = _descriptor.FileDescriptor( name='vtrpc.proto', package='vtrpc', + syntax='proto3', serialized_pb=_b('\n\x0bvtrpc.proto\x12\x05vtrpc\"F\n\x08\x43\x61llerID\x12\x11\n\tprincipal\x18\x01 \x01(\t\x12\x11\n\tcomponent\x18\x02 \x01(\t\x12\x14\n\x0csubcomponent\x18\x03 \x01(\t\"E\n\x08RPCError\x12(\n\x04\x63ode\x18\x01 \x01(\x0e\x32\x1a.vtrpc.ErrorCodeDeprecated\x12\x0f\n\x07message\x18\x02 \x01(\t*\x80\x02\n\tErrorCode\x12\x0b\n\x07SUCCESS\x10\x00\x12\r\n\tCANCELLED\x10\x01\x12\x11\n\rUNKNOWN_ERROR\x10\x02\x12\r\n\tBAD_INPUT\x10\x03\x12\x15\n\x11\x44\x45\x41\x44LINE_EXCEEDED\x10\x04\x12\x13\n\x0fINTEGRITY_ERROR\x10\x05\x12\x15\n\x11PERMISSION_DENIED\x10\x06\x12\x13\n\x0fTHROTTLED_ERROR\x10\x07\x12\x14\n\x10QUERY_NOT_SERVED\x10\x08\x12\r\n\tNOT_IN_TX\x10\t\x12\x12\n\x0eINTERNAL_ERROR\x10\n\x12$\n RESOURCE_TEMPORARILY_UNAVAILABLE\x10\x0b*x\n\x13\x45rrorCodeDeprecated\x12\x0b\n\x07NoError\x10\x00\x12\x10\n\x0bTabletError\x10\xe8\x07\x12\x17\n\x12UnknownTabletError\x10\xcf\x0f\x12\x10\n\x0bVtgateError\x10\xd0\x0f\x12\x17\n\x12UnknownVtgateError\x10\xb7\x17\x42\x1a\n\x18\x63om.youtube.vitess.protob\x06proto3') ) _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -177,6 +178,7 @@ _CALLERID = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -214,6 +216,7 @@ _RPCERROR = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], diff --git a/py/vtproto/vtworkerdata_pb2.py b/py/vtproto/vtworkerdata_pb2.py index 20710ccbe1..50eaf0f897 100644 --- a/py/vtproto/vtworkerdata_pb2.py +++ b/py/vtproto/vtworkerdata_pb2.py @@ -19,6 +19,7 @@ import logutil_pb2 as logutil__pb2 DESCRIPTOR = _descriptor.FileDescriptor( name='vtworkerdata.proto', package='vtworkerdata', + syntax='proto3', serialized_pb=_b('\n\x12vtworkerdata.proto\x12\x0cvtworkerdata\x1a\rlogutil.proto\"-\n\x1d\x45xecuteVtworkerCommandRequest\x12\x0c\n\x04\x61rgs\x18\x01 \x03(\t\"?\n\x1e\x45xecuteVtworkerCommandResponse\x12\x1d\n\x05\x65vent\x18\x01 \x01(\x0b\x32\x0e.logutil.Eventb\x06proto3') , dependencies=[logutil__pb2.DESCRIPTOR,]) @@ -49,6 +50,7 @@ _EXECUTEVTWORKERCOMMANDREQUEST = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], @@ -79,6 +81,7 @@ _EXECUTEVTWORKERCOMMANDRESPONSE = _descriptor.Descriptor( ], options=None, is_extendable=False, + syntax='proto3', extension_ranges=[], oneofs=[ ], diff --git a/py/vtproto/vtworkerservice_pb2.py b/py/vtproto/vtworkerservice_pb2.py index cb34ad6388..2720b90e90 100644 --- a/py/vtproto/vtworkerservice_pb2.py +++ b/py/vtproto/vtworkerservice_pb2.py @@ -19,6 +19,7 @@ import vtworkerdata_pb2 as vtworkerdata__pb2 DESCRIPTOR = _descriptor.FileDescriptor( name='vtworkerservice.proto', package='vtworkerservice', + syntax='proto3', serialized_pb=_b('\n\x15vtworkerservice.proto\x12\x0fvtworkerservice\x1a\x12vtworkerdata.proto2\x83\x01\n\x08Vtworker\x12w\n\x16\x45xecuteVtworkerCommand\x12+.vtworkerdata.ExecuteVtworkerCommandRequest\x1a,.vtworkerdata.ExecuteVtworkerCommandResponse\"\x00\x30\x01\x62\x06proto3') , dependencies=[vtworkerdata__pb2.DESCRIPTOR,]) @@ -53,7 +54,7 @@ class EarlyAdopterVtworkerStub(object): def ExecuteVtworkerCommand(self, request): raise NotImplementedError() ExecuteVtworkerCommand.async = None -def early_adopter_create_Vtworker_server(servicer, port, root_certificates, key_chain_pairs): +def early_adopter_create_Vtworker_server(servicer, port, private_key=None, certificate_chain=None): import vtworkerdata_pb2 import vtworkerdata_pb2 method_service_descriptions = { @@ -63,8 +64,8 @@ def early_adopter_create_Vtworker_server(servicer, port, root_certificates, key_ vtworkerdata_pb2.ExecuteVtworkerCommandResponse.SerializeToString, ), } - return implementations.secure_server("vtworkerservice.Vtworker", method_service_descriptions, port, root_certificates, key_chain_pairs) -def early_adopter_create_Vtworker_stub(host, port): + return implementations.server("vtworkerservice.Vtworker", method_service_descriptions, port, private_key=private_key, certificate_chain=certificate_chain) +def early_adopter_create_Vtworker_stub(host, port, metadata_transformer=None, secure=False, root_certificates=None, private_key=None, certificate_chain=None, server_host_override=None): import vtworkerdata_pb2 import vtworkerdata_pb2 method_invocation_descriptions = { @@ -73,5 +74,5 @@ def early_adopter_create_Vtworker_stub(host, port): vtworkerdata_pb2.ExecuteVtworkerCommandResponse.FromString, ), } - return implementations.insecure_stub("vtworkerservice.Vtworker", method_invocation_descriptions, host, port) + return implementations.stub("vtworkerservice.Vtworker", method_invocation_descriptions, host, port, metadata_transformer=metadata_transformer, secure=secure, root_certificates=root_certificates, private_key=private_key, certificate_chain=certificate_chain, server_host_override=server_host_override) # @@protoc_insertion_point(module_scope) diff --git a/travis/install_grpc.sh b/travis/install_grpc.sh index 3fd97a6873..8a8f3c61e6 100755 --- a/travis/install_grpc.sh +++ b/travis/install_grpc.sh @@ -13,7 +13,7 @@ fi git clone https://github.com/grpc/grpc.git cd grpc -git checkout 1528ba54d71ab477266c9ff4235446445d5de1c1 +git checkout 7b8fd39eabc3429ce8cb5fd6b97ccb278bc2dda6 git submodule update --init make if [ "$grpc_dist" != "" ]; then diff --git a/travis/install_protobuf.sh b/travis/install_protobuf.sh index 7ec316f310..6ecd1e1e0a 100755 --- a/travis/install_protobuf.sh +++ b/travis/install_protobuf.sh @@ -11,9 +11,9 @@ if [ "$protobuf_dist" != "" ]; then cd $protobuf_dist fi -wget https://github.com/google/protobuf/archive/v3.0.0-alpha-2.tar.gz -tar -xvzf v3.0.0-alpha-2.tar.gz -cd protobuf-3.0.0-alpha-2 +wget https://github.com/google/protobuf/archive/v3.0.0-alpha-3.tar.gz +tar -xvzf v3.0.0-alpha-3.tar.gz +cd protobuf-3.0.0-alpha-3 ./autogen.sh if [ "$protobuf_dist" != "" ]; then ./configure --prefix=$protobuf_dist From b231b69851cfcf9d9bb761539fe769678aa75419 Mon Sep 17 00:00:00 2001 From: Michael Berlin Date: Tue, 18 Aug 2015 13:06:52 -0700 Subject: [PATCH 37/80] proto: Reorder fields in RealTimeStats. --- go/vt/proto/query/query.pb.go | 12 ++++++------ proto/query.proto | 14 +++++++------- py/vtproto/query_pb2.py | 10 +++++----- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/go/vt/proto/query/query.pb.go b/go/vt/proto/query/query.pb.go index 4ba7f4972f..45ab67f23b 100644 --- a/go/vt/proto/query/query.pb.go +++ b/go/vt/proto/query/query.pb.go @@ -805,17 +805,17 @@ type RealtimeStats struct { // NOTE: This field must not be evaluated if "health_error" is not empty. // TODO(mberlin): Let's switch it to int64 instead? SecondsBehindMaster uint32 `protobuf:"varint,2,opt,name=seconds_behind_master" json:"seconds_behind_master,omitempty"` + // bin_log_players_count is the number of currently running binlog players. + // if the value is 0, it means that filtered replication is currently not + // running on the tablet. If >0, filtered replication is running. + // NOTE: This field must not be evaluated if "health_error" is not empty. + BinlogPlayersCount int32 `protobuf:"varint,3,opt,name=binlog_players_count" json:"binlog_players_count,omitempty"` // seconds_behind_master_filtered_replication is populated for the receiving // master of an ongoing filtered replication only. // It specifies how far the receiving master lags behind the sending master. // NOTE: This field must not be evaluated if "health_error" is not empty. // NOTE: This field must not be evaluated if "bin_log_players_count" is 0. - SecondsBehindMasterFilteredReplication int64 `protobuf:"varint,3,opt,name=seconds_behind_master_filtered_replication" json:"seconds_behind_master_filtered_replication,omitempty"` - // bin_log_players_count is the number of currently running binlog players. - // if the value is 0, it means that filtered replication is currently not - // running on the tablet. If >0, filtered replication is running. - // NOTE: This field must not be evaluated if "health_error" is not empty. - BinlogPlayersCount int32 `protobuf:"varint,4,opt,name=binlog_players_count" json:"binlog_players_count,omitempty"` + SecondsBehindMasterFilteredReplication int64 `protobuf:"varint,4,opt,name=seconds_behind_master_filtered_replication" json:"seconds_behind_master_filtered_replication,omitempty"` // cpu_usage is used for load-based balancing CpuUsage float64 `protobuf:"fixed64,5,opt,name=cpu_usage" json:"cpu_usage,omitempty"` } diff --git a/proto/query.proto b/proto/query.proto index ad88adaba0..c4c36156ac 100644 --- a/proto/query.proto +++ b/proto/query.proto @@ -289,18 +289,18 @@ message RealtimeStats { // TODO(mberlin): Let's switch it to int64 instead? uint32 seconds_behind_master = 2; + // bin_log_players_count is the number of currently running binlog players. + // if the value is 0, it means that filtered replication is currently not + // running on the tablet. If >0, filtered replication is running. + // NOTE: This field must not be evaluated if "health_error" is not empty. + int32 binlog_players_count = 3; + // seconds_behind_master_filtered_replication is populated for the receiving // master of an ongoing filtered replication only. // It specifies how far the receiving master lags behind the sending master. // NOTE: This field must not be evaluated if "health_error" is not empty. // NOTE: This field must not be evaluated if "bin_log_players_count" is 0. - int64 seconds_behind_master_filtered_replication = 3; - - // bin_log_players_count is the number of currently running binlog players. - // if the value is 0, it means that filtered replication is currently not - // running on the tablet. If >0, filtered replication is running. - // NOTE: This field must not be evaluated if "health_error" is not empty. - int32 binlog_players_count = 4; + int64 seconds_behind_master_filtered_replication = 4; // cpu_usage is used for load-based balancing double cpu_usage = 5; diff --git a/py/vtproto/query_pb2.py b/py/vtproto/query_pb2.py index a2947063b2..e25f0146ec 100644 --- a/py/vtproto/query_pb2.py +++ b/py/vtproto/query_pb2.py @@ -21,7 +21,7 @@ DESCRIPTOR = _descriptor.FileDescriptor( name='query.proto', package='query', syntax='proto3', - serialized_pb=_b('\n\x0bquery.proto\x12\x05query\x1a\x0etopodata.proto\x1a\x0bvtrpc.proto\"T\n\x06Target\x12\x10\n\x08keyspace\x18\x01 \x01(\t\x12\r\n\x05shard\x18\x02 \x01(\t\x12)\n\x0btablet_type\x18\x03 \x01(\x0e\x32\x14.topodata.TabletType\"\"\n\x0eVTGateCallerID\x12\x10\n\x08username\x18\x01 \x01(\t\"\x92\x03\n\x0c\x42indVariable\x12&\n\x04type\x18\x01 \x01(\x0e\x32\x18.query.BindVariable.Type\x12\x13\n\x0bvalue_bytes\x18\x02 \x01(\x0c\x12\x11\n\tvalue_int\x18\x03 \x01(\x03\x12\x12\n\nvalue_uint\x18\x04 \x01(\x04\x12\x13\n\x0bvalue_float\x18\x05 \x01(\x01\x12\x18\n\x10value_bytes_list\x18\x06 \x03(\x0c\x12\x16\n\x0evalue_int_list\x18\x07 \x03(\x03\x12\x17\n\x0fvalue_uint_list\x18\x08 \x03(\x04\x12\x18\n\x10value_float_list\x18\t \x03(\x01\"\xa3\x01\n\x04Type\x12\r\n\tTYPE_NULL\x10\x00\x12\x0e\n\nTYPE_BYTES\x10\x01\x12\x0c\n\x08TYPE_INT\x10\x02\x12\r\n\tTYPE_UINT\x10\x03\x12\x0e\n\nTYPE_FLOAT\x10\x04\x12\x13\n\x0fTYPE_BYTES_LIST\x10\x05\x12\x11\n\rTYPE_INT_LIST\x10\x06\x12\x12\n\x0eTYPE_UINT_LIST\x10\x07\x12\x13\n\x0fTYPE_FLOAT_LIST\x10\x08\"\xa2\x01\n\nBoundQuery\x12\x0b\n\x03sql\x18\x01 \x01(\x0c\x12<\n\x0e\x62ind_variables\x18\x02 \x03(\x0b\x32$.query.BoundQuery.BindVariablesEntry\x1aI\n\x12\x42indVariablesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\"\n\x05value\x18\x02 \x01(\x0b\x32\x13.query.BindVariable:\x02\x38\x01\"\xa1\x07\n\x05\x46ield\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x1f\n\x04type\x18\x02 \x01(\x0e\x32\x11.query.Field.Type\x12\r\n\x05\x66lags\x18\x03 \x01(\x03\"\xe1\x03\n\x04Type\x12\x10\n\x0cTYPE_DECIMAL\x10\x00\x12\r\n\tTYPE_TINY\x10\x01\x12\x0e\n\nTYPE_SHORT\x10\x02\x12\r\n\tTYPE_LONG\x10\x03\x12\x0e\n\nTYPE_FLOAT\x10\x04\x12\x0f\n\x0bTYPE_DOUBLE\x10\x05\x12\r\n\tTYPE_NULL\x10\x06\x12\x12\n\x0eTYPE_TIMESTAMP\x10\x07\x12\x11\n\rTYPE_LONGLONG\x10\x08\x12\x0e\n\nTYPE_INT24\x10\t\x12\r\n\tTYPE_DATE\x10\n\x12\r\n\tTYPE_TIME\x10\x0b\x12\x11\n\rTYPE_DATETIME\x10\x0c\x12\r\n\tTYPE_YEAR\x10\r\x12\x10\n\x0cTYPE_NEWDATE\x10\x0e\x12\x10\n\x0cTYPE_VARCHAR\x10\x0f\x12\x0c\n\x08TYPE_BIT\x10\x10\x12\x14\n\x0fTYPE_NEWDECIMAL\x10\xf6\x01\x12\x0e\n\tTYPE_ENUM\x10\xf7\x01\x12\r\n\x08TYPE_SET\x10\xf8\x01\x12\x13\n\x0eTYPE_TINY_BLOB\x10\xf9\x01\x12\x15\n\x10TYPE_MEDIUM_BLOB\x10\xfa\x01\x12\x13\n\x0eTYPE_LONG_BLOB\x10\xfb\x01\x12\x0e\n\tTYPE_BLOB\x10\xfc\x01\x12\x14\n\x0fTYPE_VAR_STRING\x10\xfd\x01\x12\x10\n\x0bTYPE_STRING\x10\xfe\x01\x12\x12\n\rTYPE_GEOMETRY\x10\xff\x01\"\xf5\x02\n\x04\x46lag\x12\x15\n\x11VT_ZEROVALUE_FLAG\x10\x00\x12\x14\n\x10VT_NOT_NULL_FLAG\x10\x01\x12\x13\n\x0fVT_PRI_KEY_FLAG\x10\x02\x12\x16\n\x12VT_UNIQUE_KEY_FLAG\x10\x04\x12\x18\n\x14VT_MULTIPLE_KEY_FLAG\x10\x08\x12\x10\n\x0cVT_BLOB_FLAG\x10\x10\x12\x14\n\x10VT_UNSIGNED_FLAG\x10 \x12\x14\n\x10VT_ZEROFILL_FLAG\x10@\x12\x13\n\x0eVT_BINARY_FLAG\x10\x80\x01\x12\x11\n\x0cVT_ENUM_FLAG\x10\x80\x02\x12\x1b\n\x16VT_AUTO_INCREMENT_FLAG\x10\x80\x04\x12\x16\n\x11VT_TIMESTAMP_FLAG\x10\x80\x08\x12\x10\n\x0bVT_SET_FLAG\x10\x80\x10\x12\x1d\n\x18VT_NO_DEFAULT_VALUE_FLAG\x10\x80 \x12\x1a\n\x15VT_ON_UPDATE_NOW_FLAG\x10\x80@\x12\x11\n\x0bVT_NUM_FLAG\x10\x80\x80\x02\"\x15\n\x03Row\x12\x0e\n\x06values\x18\x01 \x03(\x0c\"o\n\x0bQueryResult\x12\x1c\n\x06\x66ields\x18\x01 \x03(\x0b\x32\x0c.query.Field\x12\x15\n\rrows_affected\x18\x02 \x01(\x04\x12\x11\n\tinsert_id\x18\x03 \x01(\x04\x12\x18\n\x04rows\x18\x04 \x03(\x0b\x32\n.query.Row\"\x98\x01\n\x13GetSessionIdRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x10\n\x08keyspace\x18\x03 \x01(\t\x12\r\n\x05shard\x18\x04 \x01(\t\"*\n\x14GetSessionIdResponse\x12\x12\n\nsession_id\x18\x01 \x01(\x03\"\xdf\x01\n\x0e\x45xecuteRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12 \n\x05query\x18\x04 \x01(\x0b\x32\x11.query.BoundQuery\x12\x16\n\x0etransaction_id\x18\x05 \x01(\x03\x12\x12\n\nsession_id\x18\x06 \x01(\x03\"5\n\x0f\x45xecuteResponse\x12\"\n\x06result\x18\x01 \x01(\x0b\x32\x12.query.QueryResult\"\xfe\x01\n\x13\x45xecuteBatchRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12\"\n\x07queries\x18\x04 \x03(\x0b\x32\x11.query.BoundQuery\x12\x16\n\x0e\x61s_transaction\x18\x05 \x01(\x08\x12\x16\n\x0etransaction_id\x18\x06 \x01(\x03\x12\x12\n\nsession_id\x18\x07 \x01(\x03\";\n\x14\x45xecuteBatchResponse\x12#\n\x07results\x18\x01 \x03(\x0b\x32\x12.query.QueryResult\"\xcd\x01\n\x14StreamExecuteRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12 \n\x05query\x18\x04 \x01(\x0b\x32\x11.query.BoundQuery\x12\x12\n\nsession_id\x18\x05 \x01(\x03\";\n\x15StreamExecuteResponse\x12\"\n\x06result\x18\x01 \x01(\x0b\x32\x12.query.QueryResult\"\xa3\x01\n\x0c\x42\x65ginRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12\x12\n\nsession_id\x18\x04 \x01(\x03\"\'\n\rBeginResponse\x12\x16\n\x0etransaction_id\x18\x01 \x01(\x03\"\xbc\x01\n\rCommitRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12\x16\n\x0etransaction_id\x18\x04 \x01(\x03\x12\x12\n\nsession_id\x18\x05 \x01(\x03\"\x10\n\x0e\x43ommitResponse\"\xbe\x01\n\x0fRollbackRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12\x16\n\x0etransaction_id\x18\x04 \x01(\x03\x12\x12\n\nsession_id\x18\x05 \x01(\x03\"\x12\n\x10RollbackResponse\"\xf5\x01\n\x11SplitQueryRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12 \n\x05query\x18\x04 \x01(\x0b\x32\x11.query.BoundQuery\x12\x14\n\x0csplit_column\x18\x05 \x01(\t\x12\x13\n\x0bsplit_count\x18\x06 \x01(\x03\x12\x12\n\nsession_id\x18\x07 \x01(\x03\"A\n\nQuerySplit\x12 \n\x05query\x18\x01 \x01(\x0b\x32\x11.query.BoundQuery\x12\x11\n\trow_count\x18\x02 \x01(\x03\"8\n\x12SplitQueryResponse\x12\"\n\x07queries\x18\x01 \x03(\x0b\x32\x11.query.QuerySplit\"\x15\n\x13StreamHealthRequest\"\xa9\x01\n\rRealtimeStats\x12\x14\n\x0chealth_error\x18\x01 \x01(\t\x12\x1d\n\x15seconds_behind_master\x18\x02 \x01(\r\x12\x32\n*seconds_behind_master_filtered_replication\x18\x03 \x01(\x03\x12\x1c\n\x14\x62inlog_players_count\x18\x04 \x01(\x05\x12\x11\n\tcpu_usage\x18\x05 \x01(\x01\"\x93\x01\n\x14StreamHealthResponse\x12\x1d\n\x06target\x18\x01 \x01(\x0b\x32\r.query.Target\x12.\n&tablet_externally_reparented_timestamp\x18\x02 \x01(\x03\x12,\n\x0erealtime_stats\x18\x03 \x01(\x0b\x32\x14.query.RealtimeStatsB\x1a\n\x18\x63om.youtube.vitess.protob\x06proto3') + serialized_pb=_b('\n\x0bquery.proto\x12\x05query\x1a\x0etopodata.proto\x1a\x0bvtrpc.proto\"T\n\x06Target\x12\x10\n\x08keyspace\x18\x01 \x01(\t\x12\r\n\x05shard\x18\x02 \x01(\t\x12)\n\x0btablet_type\x18\x03 \x01(\x0e\x32\x14.topodata.TabletType\"\"\n\x0eVTGateCallerID\x12\x10\n\x08username\x18\x01 \x01(\t\"\x92\x03\n\x0c\x42indVariable\x12&\n\x04type\x18\x01 \x01(\x0e\x32\x18.query.BindVariable.Type\x12\x13\n\x0bvalue_bytes\x18\x02 \x01(\x0c\x12\x11\n\tvalue_int\x18\x03 \x01(\x03\x12\x12\n\nvalue_uint\x18\x04 \x01(\x04\x12\x13\n\x0bvalue_float\x18\x05 \x01(\x01\x12\x18\n\x10value_bytes_list\x18\x06 \x03(\x0c\x12\x16\n\x0evalue_int_list\x18\x07 \x03(\x03\x12\x17\n\x0fvalue_uint_list\x18\x08 \x03(\x04\x12\x18\n\x10value_float_list\x18\t \x03(\x01\"\xa3\x01\n\x04Type\x12\r\n\tTYPE_NULL\x10\x00\x12\x0e\n\nTYPE_BYTES\x10\x01\x12\x0c\n\x08TYPE_INT\x10\x02\x12\r\n\tTYPE_UINT\x10\x03\x12\x0e\n\nTYPE_FLOAT\x10\x04\x12\x13\n\x0fTYPE_BYTES_LIST\x10\x05\x12\x11\n\rTYPE_INT_LIST\x10\x06\x12\x12\n\x0eTYPE_UINT_LIST\x10\x07\x12\x13\n\x0fTYPE_FLOAT_LIST\x10\x08\"\xa2\x01\n\nBoundQuery\x12\x0b\n\x03sql\x18\x01 \x01(\x0c\x12<\n\x0e\x62ind_variables\x18\x02 \x03(\x0b\x32$.query.BoundQuery.BindVariablesEntry\x1aI\n\x12\x42indVariablesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\"\n\x05value\x18\x02 \x01(\x0b\x32\x13.query.BindVariable:\x02\x38\x01\"\xa1\x07\n\x05\x46ield\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x1f\n\x04type\x18\x02 \x01(\x0e\x32\x11.query.Field.Type\x12\r\n\x05\x66lags\x18\x03 \x01(\x03\"\xe1\x03\n\x04Type\x12\x10\n\x0cTYPE_DECIMAL\x10\x00\x12\r\n\tTYPE_TINY\x10\x01\x12\x0e\n\nTYPE_SHORT\x10\x02\x12\r\n\tTYPE_LONG\x10\x03\x12\x0e\n\nTYPE_FLOAT\x10\x04\x12\x0f\n\x0bTYPE_DOUBLE\x10\x05\x12\r\n\tTYPE_NULL\x10\x06\x12\x12\n\x0eTYPE_TIMESTAMP\x10\x07\x12\x11\n\rTYPE_LONGLONG\x10\x08\x12\x0e\n\nTYPE_INT24\x10\t\x12\r\n\tTYPE_DATE\x10\n\x12\r\n\tTYPE_TIME\x10\x0b\x12\x11\n\rTYPE_DATETIME\x10\x0c\x12\r\n\tTYPE_YEAR\x10\r\x12\x10\n\x0cTYPE_NEWDATE\x10\x0e\x12\x10\n\x0cTYPE_VARCHAR\x10\x0f\x12\x0c\n\x08TYPE_BIT\x10\x10\x12\x14\n\x0fTYPE_NEWDECIMAL\x10\xf6\x01\x12\x0e\n\tTYPE_ENUM\x10\xf7\x01\x12\r\n\x08TYPE_SET\x10\xf8\x01\x12\x13\n\x0eTYPE_TINY_BLOB\x10\xf9\x01\x12\x15\n\x10TYPE_MEDIUM_BLOB\x10\xfa\x01\x12\x13\n\x0eTYPE_LONG_BLOB\x10\xfb\x01\x12\x0e\n\tTYPE_BLOB\x10\xfc\x01\x12\x14\n\x0fTYPE_VAR_STRING\x10\xfd\x01\x12\x10\n\x0bTYPE_STRING\x10\xfe\x01\x12\x12\n\rTYPE_GEOMETRY\x10\xff\x01\"\xf5\x02\n\x04\x46lag\x12\x15\n\x11VT_ZEROVALUE_FLAG\x10\x00\x12\x14\n\x10VT_NOT_NULL_FLAG\x10\x01\x12\x13\n\x0fVT_PRI_KEY_FLAG\x10\x02\x12\x16\n\x12VT_UNIQUE_KEY_FLAG\x10\x04\x12\x18\n\x14VT_MULTIPLE_KEY_FLAG\x10\x08\x12\x10\n\x0cVT_BLOB_FLAG\x10\x10\x12\x14\n\x10VT_UNSIGNED_FLAG\x10 \x12\x14\n\x10VT_ZEROFILL_FLAG\x10@\x12\x13\n\x0eVT_BINARY_FLAG\x10\x80\x01\x12\x11\n\x0cVT_ENUM_FLAG\x10\x80\x02\x12\x1b\n\x16VT_AUTO_INCREMENT_FLAG\x10\x80\x04\x12\x16\n\x11VT_TIMESTAMP_FLAG\x10\x80\x08\x12\x10\n\x0bVT_SET_FLAG\x10\x80\x10\x12\x1d\n\x18VT_NO_DEFAULT_VALUE_FLAG\x10\x80 \x12\x1a\n\x15VT_ON_UPDATE_NOW_FLAG\x10\x80@\x12\x11\n\x0bVT_NUM_FLAG\x10\x80\x80\x02\"\x15\n\x03Row\x12\x0e\n\x06values\x18\x01 \x03(\x0c\"o\n\x0bQueryResult\x12\x1c\n\x06\x66ields\x18\x01 \x03(\x0b\x32\x0c.query.Field\x12\x15\n\rrows_affected\x18\x02 \x01(\x04\x12\x11\n\tinsert_id\x18\x03 \x01(\x04\x12\x18\n\x04rows\x18\x04 \x03(\x0b\x32\n.query.Row\"\x98\x01\n\x13GetSessionIdRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x10\n\x08keyspace\x18\x03 \x01(\t\x12\r\n\x05shard\x18\x04 \x01(\t\"*\n\x14GetSessionIdResponse\x12\x12\n\nsession_id\x18\x01 \x01(\x03\"\xdf\x01\n\x0e\x45xecuteRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12 \n\x05query\x18\x04 \x01(\x0b\x32\x11.query.BoundQuery\x12\x16\n\x0etransaction_id\x18\x05 \x01(\x03\x12\x12\n\nsession_id\x18\x06 \x01(\x03\"5\n\x0f\x45xecuteResponse\x12\"\n\x06result\x18\x01 \x01(\x0b\x32\x12.query.QueryResult\"\xfe\x01\n\x13\x45xecuteBatchRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12\"\n\x07queries\x18\x04 \x03(\x0b\x32\x11.query.BoundQuery\x12\x16\n\x0e\x61s_transaction\x18\x05 \x01(\x08\x12\x16\n\x0etransaction_id\x18\x06 \x01(\x03\x12\x12\n\nsession_id\x18\x07 \x01(\x03\";\n\x14\x45xecuteBatchResponse\x12#\n\x07results\x18\x01 \x03(\x0b\x32\x12.query.QueryResult\"\xcd\x01\n\x14StreamExecuteRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12 \n\x05query\x18\x04 \x01(\x0b\x32\x11.query.BoundQuery\x12\x12\n\nsession_id\x18\x05 \x01(\x03\";\n\x15StreamExecuteResponse\x12\"\n\x06result\x18\x01 \x01(\x0b\x32\x12.query.QueryResult\"\xa3\x01\n\x0c\x42\x65ginRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12\x12\n\nsession_id\x18\x04 \x01(\x03\"\'\n\rBeginResponse\x12\x16\n\x0etransaction_id\x18\x01 \x01(\x03\"\xbc\x01\n\rCommitRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12\x16\n\x0etransaction_id\x18\x04 \x01(\x03\x12\x12\n\nsession_id\x18\x05 \x01(\x03\"\x10\n\x0e\x43ommitResponse\"\xbe\x01\n\x0fRollbackRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12\x16\n\x0etransaction_id\x18\x04 \x01(\x03\x12\x12\n\nsession_id\x18\x05 \x01(\x03\"\x12\n\x10RollbackResponse\"\xf5\x01\n\x11SplitQueryRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12 \n\x05query\x18\x04 \x01(\x0b\x32\x11.query.BoundQuery\x12\x14\n\x0csplit_column\x18\x05 \x01(\t\x12\x13\n\x0bsplit_count\x18\x06 \x01(\x03\x12\x12\n\nsession_id\x18\x07 \x01(\x03\"A\n\nQuerySplit\x12 \n\x05query\x18\x01 \x01(\x0b\x32\x11.query.BoundQuery\x12\x11\n\trow_count\x18\x02 \x01(\x03\"8\n\x12SplitQueryResponse\x12\"\n\x07queries\x18\x01 \x03(\x0b\x32\x11.query.QuerySplit\"\x15\n\x13StreamHealthRequest\"\xa9\x01\n\rRealtimeStats\x12\x14\n\x0chealth_error\x18\x01 \x01(\t\x12\x1d\n\x15seconds_behind_master\x18\x02 \x01(\r\x12\x1c\n\x14\x62inlog_players_count\x18\x03 \x01(\x05\x12\x32\n*seconds_behind_master_filtered_replication\x18\x04 \x01(\x03\x12\x11\n\tcpu_usage\x18\x05 \x01(\x01\"\x93\x01\n\x14StreamHealthResponse\x12\x1d\n\x06target\x18\x01 \x01(\x0b\x32\r.query.Target\x12.\n&tablet_externally_reparented_timestamp\x18\x02 \x01(\x03\x12,\n\x0erealtime_stats\x18\x03 \x01(\x0b\x32\x14.query.RealtimeStatsB\x1a\n\x18\x63om.youtube.vitess.protob\x06proto3') , dependencies=[topodata__pb2.DESCRIPTOR,vtrpc__pb2.DESCRIPTOR,]) _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -1459,15 +1459,15 @@ _REALTIMESTATS = _descriptor.Descriptor( is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='seconds_behind_master_filtered_replication', full_name='query.RealtimeStats.seconds_behind_master_filtered_replication', index=2, - number=3, type=3, cpp_type=2, label=1, + name='binlog_players_count', full_name='query.RealtimeStats.binlog_players_count', index=2, + number=3, type=5, cpp_type=1, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='binlog_players_count', full_name='query.RealtimeStats.binlog_players_count', index=3, - number=4, type=5, cpp_type=1, label=1, + name='seconds_behind_master_filtered_replication', full_name='query.RealtimeStats.seconds_behind_master_filtered_replication', index=3, + number=4, type=3, cpp_type=2, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, From 66146f631f98d904caa622d407a69438e5d14d8b Mon Sep 17 00:00:00 2001 From: Michael Berlin Date: Tue, 18 Aug 2015 13:12:45 -0700 Subject: [PATCH 38/80] vtctl: Revert that "master" is allowed as type for RunHealthCheck. A tablet is never started with -tablet_type master and therefore the manual health check shouldn't support this as well. Nonetheless, any "replica" can become a "master". --- go/vt/vtctl/vtctl.go | 2 +- test/resharding.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/go/vt/vtctl/vtctl.go b/go/vt/vtctl/vtctl.go index 7b7cfc0f2d..9c5568b504 100644 --- a/go/vt/vtctl/vtctl.go +++ b/go/vt/vtctl/vtctl.go @@ -911,7 +911,7 @@ func commandRunHealthCheck(ctx context.Context, wr *wrangler.Wrangler, subFlags if err != nil { return err } - servedType, err := parseTabletType(subFlags.Arg(1), []pb.TabletType{pb.TabletType_MASTER, pb.TabletType_REPLICA, pb.TabletType_RDONLY}) + servedType, err := parseTabletType(subFlags.Arg(1), []pb.TabletType{pb.TabletType_REPLICA, pb.TabletType_RDONLY}) if err != nil { return err } diff --git a/test/resharding.py b/test/resharding.py index 9a0d63ad53..b9e5ede646 100755 --- a/test/resharding.py +++ b/test/resharding.py @@ -407,7 +407,7 @@ primary key (name) def _check_stream_health_equals_binlog_player_vars(self, tablet): blp_stats = utils.get_vars(tablet.port) # Enforce health check because it's not running by default as tablets are not started with it. - utils.run_vtctl(["RunHealthCheck", tablet.tablet_alias, 'master']) + utils.run_vtctl(["RunHealthCheck", tablet.tablet_alias, 'replica']) stream_health, _ = utils.run_vtctl(['VtTabletStreamHealth', '-count', '1', tablet.tablet_alias], From e87b7ffd98a0e234a1e2e5e47d43aebeb7c7fe45 Mon Sep 17 00:00:00 2001 From: Michael Berlin Date: Tue, 18 Aug 2015 13:18:08 -0700 Subject: [PATCH 39/80] topo: Generate list of tablet types in lower case. --- go/vt/topo/tablet.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/topo/tablet.go b/go/vt/topo/tablet.go index 96ba682a7b..c92eef7c87 100644 --- a/go/vt/topo/tablet.go +++ b/go/vt/topo/tablet.go @@ -235,7 +235,7 @@ func IsTypeInList(tabletType pb.TabletType, types []pb.TabletType) bool { func MakeStringTypeList(types []pb.TabletType) []string { strs := make([]string, len(types)) for i, t := range types { - strs[i] = t.String() + strs[i] = strings.ToLower(t.String()) } sort.Strings(strs) return strs From 1e37ca1a68dd84cf1688974b3765b526c2b7ae15 Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Tue, 18 Aug 2015 13:22:24 -0700 Subject: [PATCH 40/80] tabletserver: INSERT IGNORE support This feature is needed for improving the usability of ExecuteBatch. People may want a dup key error from insert to be ignored so that they can follow it with other statements. In the implementation, there's one caveat I had to handle, which is that the ignore should be removed if it's an upsert. Otherwise, we never get the dup key errror. --- data/test/tabletserver/exec_cases.txt | 4 +- go/vt/sqlparser/ast.go | 20 +- go/vt/sqlparser/sql.go | 750 ++++++++++---------- go/vt/sqlparser/sql.y | 21 +- go/vt/tabletserver/planbuilder/dml.go | 6 +- go/vt/tabletserver/planbuilder/query_gen.go | 12 +- test/queryservice_tests/nocache_cases.py | 27 +- 7 files changed, 434 insertions(+), 406 deletions(-) diff --git a/data/test/tabletserver/exec_cases.txt b/data/test/tabletserver/exec_cases.txt index d757953fcd..035271a6c3 100644 --- a/data/test/tabletserver/exec_cases.txt +++ b/data/test/tabletserver/exec_cases.txt @@ -1793,7 +1793,7 @@ "TableName": "b", "FieldQuery": null, "FullQuery": "insert into b(id, eid) values (1, 2) on duplicate key update eid = func(a)", - "OuterQuery": "insert into b(id, eid) values (1, 2)", + "OuterQuery": null, "Subquery": null, "UpsertQuery": null, "IndexUsed": "", @@ -1814,7 +1814,7 @@ "TableName": "b", "FieldQuery": null, "FullQuery": "insert into b(id, eid) values (1, 2), (2, 3) on duplicate key update name = func(a)", - "OuterQuery": "insert into b(id, eid) values (1, 2), (2, 3)", + "OuterQuery": null, "Subquery": null, "UpsertQuery": null, "IndexUsed": "", diff --git a/go/vt/sqlparser/ast.go b/go/vt/sqlparser/ast.go index c81d8e0aea..938498978d 100644 --- a/go/vt/sqlparser/ast.go +++ b/go/vt/sqlparser/ast.go @@ -48,6 +48,13 @@ func String(node SQLNode) string { return buf.String() } +// GenerateParsedQuery returns a ParsedQuery of the ast. +func GenerateParsedQuery(node SQLNode) *ParsedQuery { + buf := NewTrackedBuffer(nil) + buf.Myprintf("%v", node) + return buf.ParsedQuery() +} + // Statement represents a statement. type Statement interface { IStatement() @@ -129,6 +136,7 @@ func (node *Union) Format(buf *TrackedBuffer) { // Insert represents an INSERT statement. type Insert struct { Comments Comments + Ignore string Table *TableName Columns Columns Rows InsertRows @@ -136,8 +144,8 @@ type Insert struct { } func (node *Insert) Format(buf *TrackedBuffer) { - buf.Myprintf("insert %vinto %v%v %v%v", - node.Comments, + buf.Myprintf("insert %v%sinto %v%v %v%v", + node.Comments, node.Ignore, node.Table, node.Columns, node.Rows, node.OnDup) } @@ -397,13 +405,13 @@ type IndexHints struct { } const ( - AST_USE = "use" - AST_IGNORE = "ignore" - AST_FORCE = "force" + AST_USE = "use " + AST_IGNORE = "ignore " + AST_FORCE = "force " ) func (node *IndexHints) Format(buf *TrackedBuffer) { - buf.Myprintf(" %s index ", node.Type) + buf.Myprintf(" %sindex ", node.Type) prefix := "(" for _, n := range node.Indexes { buf.Myprintf("%s%v", prefix, n) diff --git a/go/vt/sqlparser/sql.go b/go/vt/sqlparser/sql.go index 36025c9d1b..c8fcca4be5 100644 --- a/go/vt/sqlparser/sql.go +++ b/go/vt/sqlparser/sql.go @@ -284,136 +284,134 @@ const yyPrivate = 57344 var yyTokenNames []string var yyStates []string -const yyLast = 722 +const yyLast = 707 var yyAct = [...]int{ - 98, 165, 162, 339, 235, 372, 256, 93, 302, 63, - 94, 82, 247, 203, 92, 214, 294, 183, 164, 3, - 64, 236, 267, 268, 269, 270, 271, 83, 272, 273, - 381, 28, 29, 30, 31, 107, 137, 136, 66, 131, - 238, 72, 50, 300, 75, 65, 53, 68, 104, 105, - 106, 191, 78, 38, 70, 40, 350, 103, 88, 41, - 263, 125, 319, 321, 74, 43, 87, 44, 51, 52, - 236, 349, 348, 236, 49, 108, 46, 47, 48, 236, - 71, 129, 95, 96, 236, 45, 133, 330, 236, 236, - 97, 248, 236, 110, 166, 320, 278, 135, 167, 169, - 170, 248, 118, 292, 137, 136, 114, 146, 109, 152, - 153, 146, 120, 177, 66, 122, 223, 66, 124, 187, - 186, 65, 332, 181, 65, 144, 147, 148, 149, 150, - 151, 152, 153, 146, 88, 209, 187, 185, 137, 136, - 136, 213, 345, 295, 221, 222, 259, 225, 226, 227, - 228, 229, 230, 231, 232, 233, 234, 161, 163, 208, - 224, 197, 188, 73, 128, 168, 347, 116, 116, 237, - 239, 240, 201, 346, 88, 88, 241, 173, 295, 195, - 66, 66, 238, 242, 244, 198, 317, 65, 254, 252, - 245, 59, 260, 313, 316, 315, 311, 251, 314, 255, - 210, 312, 130, 211, 212, 357, 207, 184, 184, 356, - 28, 29, 30, 31, 77, 216, 334, 279, 240, 277, - 264, 91, 281, 282, 145, 144, 147, 148, 149, 150, - 151, 152, 153, 146, 280, 261, 289, 194, 196, 193, - 117, 285, 103, 14, 111, 73, 88, 286, 131, 288, - 112, 258, 265, 116, 115, 287, 91, 91, 299, 291, - 61, 297, 301, 80, 171, 172, 68, 174, 175, 298, - 103, 217, 359, 360, 179, 61, 61, 103, 180, 309, - 310, 207, 323, 378, 325, 103, 327, 121, 79, 103, - 113, 276, 328, 354, 216, 331, 205, 91, 107, 333, - 379, 66, 91, 91, 329, 215, 337, 340, 335, 61, - 336, 104, 105, 106, 134, 293, 145, 144, 147, 148, - 149, 150, 151, 152, 153, 146, 58, 284, 218, 351, - 219, 220, 73, 14, 352, 353, 189, 91, 91, 385, - 127, 207, 207, 56, 54, 355, 363, 240, 341, 361, - 91, 147, 148, 149, 150, 151, 152, 153, 146, 369, - 340, 370, 303, 250, 344, 67, 373, 373, 373, 66, - 371, 205, 374, 375, 304, 257, 65, 380, 376, 382, - 383, 343, 386, 308, 215, 184, 387, 62, 388, 149, - 150, 151, 152, 153, 146, 384, 368, 243, 14, 101, - 33, 60, 274, 132, 107, 190, 39, 262, 192, 91, - 362, 76, 364, 102, 91, 81, 68, 104, 105, 106, - 42, 86, 69, 253, 60, 178, 103, 377, 60, 358, - 338, 205, 205, 342, 307, 119, 14, 15, 16, 17, - 123, 290, 176, 126, 246, 100, 99, 296, 90, 249, - 138, 95, 96, 84, 267, 268, 269, 270, 271, 97, - 272, 273, 110, 18, 326, 89, 145, 144, 147, 148, - 149, 150, 151, 152, 153, 146, 318, 109, 204, 266, - 182, 202, 85, 236, 55, 27, 57, 13, 12, 11, - 10, 199, 9, 8, 200, 7, 206, 86, 145, 144, - 147, 148, 149, 150, 151, 152, 153, 146, 6, 91, - 5, 91, 4, 2, 365, 366, 367, 1, 0, 19, - 20, 22, 21, 23, 0, 101, 14, 0, 0, 0, - 107, 32, 24, 25, 26, 0, 0, 86, 86, 102, - 0, 101, 68, 104, 105, 106, 107, 34, 35, 36, - 37, 0, 103, 0, 0, 102, 0, 0, 68, 104, - 105, 106, 0, 0, 0, 0, 0, 0, 103, 0, - 275, 206, 0, 0, 90, 0, 0, 95, 96, 84, - 0, 0, 0, 0, 0, 97, 0, 0, 110, 0, - 90, 0, 0, 95, 96, 0, 0, 101, 0, 0, - 0, 97, 107, 109, 110, 0, 14, 0, 0, 86, - 0, 102, 0, 0, 68, 104, 105, 106, 0, 109, - 0, 0, 0, 0, 103, 305, 107, 0, 306, 0, - 0, 206, 206, 0, 0, 0, 0, 0, 68, 104, - 105, 106, 322, 0, 324, 0, 90, 0, 103, 95, - 96, 0, 0, 0, 0, 0, 0, 97, 0, 0, - 110, 0, 0, 139, 143, 141, 142, 0, 0, 0, - 0, 0, 0, 95, 96, 109, 0, 0, 0, 0, - 0, 97, 0, 0, 110, 157, 158, 159, 160, 0, - 154, 155, 156, 0, 0, 0, 0, 0, 283, 109, + 98, 296, 162, 372, 93, 87, 337, 83, 63, 212, + 165, 92, 181, 251, 64, 327, 245, 59, 94, 233, + 381, 201, 234, 164, 3, 333, 82, 262, 263, 264, + 265, 266, 189, 267, 268, 107, 137, 136, 66, 131, + 236, 72, 65, 78, 75, 70, 44, 68, 104, 105, + 106, 53, 50, 28, 29, 30, 31, 103, 88, 38, + 258, 40, 313, 315, 125, 41, 348, 74, 43, 234, + 44, 108, 347, 234, 234, 234, 346, 71, 51, 52, + 115, 129, 95, 96, 49, 45, 133, 195, 234, 234, + 97, 137, 136, 110, 166, 314, 161, 163, 167, 169, + 170, 46, 47, 48, 324, 193, 234, 246, 109, 326, + 246, 196, 287, 177, 66, 273, 135, 66, 65, 185, + 184, 65, 179, 118, 114, 122, 14, 146, 124, 73, + 178, 183, 137, 136, 88, 207, 185, 120, 136, 206, + 345, 211, 209, 210, 219, 220, 107, 223, 224, 225, + 226, 227, 228, 229, 230, 231, 232, 343, 68, 104, + 105, 106, 221, 192, 194, 191, 208, 168, 103, 328, + 254, 238, 186, 173, 88, 88, 149, 150, 151, 152, + 153, 146, 199, 128, 235, 237, 344, 243, 253, 116, + 255, 239, 250, 95, 96, 152, 153, 146, 240, 242, + 328, 97, 205, 307, 110, 305, 222, 311, 308, 310, + 306, 214, 355, 259, 309, 116, 238, 236, 182, 109, + 276, 277, 356, 91, 274, 272, 275, 145, 144, 147, + 148, 149, 150, 151, 152, 153, 146, 353, 130, 280, + 182, 284, 77, 256, 88, 28, 29, 30, 31, 66, + 66, 117, 288, 65, 294, 61, 103, 292, 91, 91, + 281, 286, 283, 260, 295, 103, 171, 172, 282, 174, + 175, 291, 215, 73, 248, 205, 111, 271, 103, 113, + 14, 321, 303, 304, 131, 116, 68, 61, 214, 103, + 325, 80, 317, 134, 319, 61, 323, 334, 203, 91, + 335, 338, 322, 121, 91, 91, 79, 213, 331, 330, + 290, 73, 367, 332, 144, 147, 148, 149, 150, 151, + 152, 153, 146, 349, 262, 263, 264, 265, 266, 351, + 267, 268, 205, 205, 66, 352, 14, 112, 354, 91, + 91, 42, 279, 350, 378, 238, 249, 361, 386, 363, + 91, 56, 339, 362, 360, 67, 187, 127, 369, 338, + 107, 379, 371, 370, 54, 373, 373, 373, 61, 374, + 375, 203, 368, 104, 105, 106, 58, 66, 103, 297, + 342, 65, 387, 298, 213, 384, 252, 388, 341, 389, + 302, 60, 380, 182, 382, 383, 216, 62, 217, 218, + 385, 76, 376, 14, 33, 81, 269, 132, 188, 91, + 39, 86, 358, 359, 91, 257, 190, 69, 60, 293, + 241, 247, 101, 377, 357, 119, 336, 107, 203, 203, + 123, 340, 301, 126, 285, 176, 102, 244, 100, 68, + 104, 105, 106, 99, 329, 289, 138, 89, 312, 103, + 14, 15, 16, 17, 202, 261, 145, 144, 147, 148, + 149, 150, 151, 152, 153, 146, 200, 85, 60, 55, + 180, 90, 27, 57, 95, 96, 84, 18, 13, 12, + 11, 197, 97, 10, 198, 110, 204, 86, 320, 9, 145, 144, 147, 148, 149, 150, 151, 152, 153, 146, - 0, 140, 145, 144, 147, 148, 149, 150, 151, 152, - 153, 146, + 109, 8, 7, 101, 6, 5, 234, 4, 107, 91, + 2, 91, 1, 0, 364, 365, 366, 102, 0, 0, + 68, 104, 105, 106, 0, 0, 0, 86, 86, 0, + 103, 0, 0, 19, 20, 22, 21, 23, 147, 148, + 149, 150, 151, 152, 153, 146, 24, 25, 26, 0, + 0, 0, 90, 0, 0, 95, 96, 84, 270, 204, + 0, 0, 0, 97, 0, 0, 110, 14, 278, 0, + 145, 144, 147, 148, 149, 150, 151, 152, 153, 146, + 0, 109, 101, 0, 0, 0, 0, 107, 0, 0, + 0, 0, 0, 0, 0, 0, 102, 86, 0, 68, + 104, 105, 106, 0, 0, 0, 101, 0, 0, 103, + 299, 107, 0, 300, 0, 0, 204, 204, 0, 0, + 102, 32, 0, 68, 104, 105, 106, 316, 0, 318, + 0, 90, 0, 103, 95, 96, 0, 34, 35, 36, + 37, 0, 97, 0, 0, 110, 0, 0, 139, 143, + 141, 142, 0, 0, 0, 90, 0, 0, 95, 96, + 109, 0, 0, 0, 0, 0, 97, 0, 0, 110, + 157, 158, 159, 160, 0, 154, 155, 156, 0, 0, + 0, 0, 0, 0, 109, 145, 144, 147, 148, 149, + 150, 151, 152, 153, 146, 0, 140, 145, 144, 147, + 148, 149, 150, 151, 152, 153, 146, } var yyPact = [...]int{ - 431, -1000, -1000, 159, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -40, - -30, -8, -17, -19, -1000, -1000, -1000, 393, 327, -1000, - -1000, -1000, 325, -1000, 298, 239, 378, 229, -44, -14, - 208, -1000, -29, 208, -1000, 239, -46, 251, -46, 239, - -1000, -1000, -1000, -1000, -1000, 505, -1000, 202, 239, 258, - 24, -1000, 239, 113, -1000, 192, -1000, 20, -1000, 239, - 43, 250, -1000, -1000, 239, -1000, -35, 239, 320, 98, - 208, -1000, 193, -1000, -1000, 295, 15, 71, 642, -1000, - 577, 521, -1000, -1000, -1000, 10, 10, 10, 195, 195, - -1000, 195, 195, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - 10, -1000, 242, 229, 239, 375, 229, 10, 208, -1000, - 316, -49, -1000, 148, -1000, 239, -1000, -1000, 239, -1000, - 223, 505, -1000, -1000, 208, 126, 577, 577, 10, 230, - 307, 10, 10, 91, 10, 10, 10, 10, 10, 10, + 445, -1000, -1000, 194, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -34, + -27, -8, 8, -9, -1000, -1000, -1000, 398, 347, -1000, + -1000, -1000, 333, -1000, -51, 250, 388, 249, -53, -17, + 236, -1000, -26, 236, -1000, 250, -55, 269, -55, 250, + -1000, -1000, -1000, -1000, -1000, 483, -1000, 234, 309, 247, + 42, -1000, 250, 160, -1000, 203, -1000, 41, -1000, 250, + 68, 266, -1000, -1000, 250, -1000, -32, 250, 337, 117, + 236, -1000, 229, -1000, -1000, 274, 34, 65, 627, -1000, + 586, 562, -1000, -1000, -1000, 10, 10, 10, 209, 209, + -1000, 209, 209, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + 10, -1000, 250, 249, 250, 383, 249, 10, 236, -1000, + 336, -68, -1000, 74, -1000, 250, -1000, -1000, 250, -1000, + 218, 483, -1000, -1000, 236, 92, 586, 586, 10, 231, + 375, 10, 10, 137, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, 642, -31, -20, -15, 642, -1000, 601, -1000, - -1000, 379, 505, -1000, 393, 273, 7, 428, 328, 229, - 229, 198, -1000, 362, 577, -1000, 428, -1000, -1000, -1000, - 80, 208, -1000, -36, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, 197, 398, 272, 238, 14, -1000, -1000, -1000, - -1000, -1000, 72, 428, -1000, 601, -1000, -1000, 230, 10, - 10, 428, 630, -1000, 302, 279, 54, -1000, 315, 315, - 32, 32, 32, 28, 28, -1000, -1000, -1000, 10, -1000, - 428, -1000, -16, 505, -16, 181, 17, -1000, 577, 77, - 195, 159, 112, -12, -1000, 362, 347, 360, 71, 239, - -1000, -1000, 239, -1000, 372, 223, 223, -1000, -1000, 140, - 137, 139, 138, 130, -2, -1000, 239, -34, 239, -15, - -1000, 428, 396, 10, -1000, 428, -1000, -16, -1000, 273, - 0, -1000, 10, 37, -1000, 270, 161, -1000, -1000, -1000, - 229, 347, -1000, 10, 10, -1000, -1000, 369, 350, 398, - 76, -1000, 117, -1000, 110, -1000, -1000, -1000, -1000, -22, - -23, -38, -1000, -1000, -1000, -1000, 10, 428, -1000, -83, - -1000, 428, 10, 263, 195, -1000, -1000, 154, 150, -1000, - 246, -1000, 362, 577, 10, 577, -1000, -1000, 195, 195, - 195, 428, -1000, 428, 389, -1000, 10, 10, -1000, -1000, - -1000, 347, 71, 127, 71, 208, 208, 208, 229, 428, - -1000, 267, -25, -1000, -25, -25, 113, -1000, 388, 318, - -1000, 208, -1000, -1000, -1000, 208, -1000, 208, -1000, + -1000, -1000, 627, -31, 2, -15, 627, -1000, 121, -1000, + -1000, 402, 483, -1000, 398, 335, 23, 615, 242, 230, + -1000, 373, 586, -1000, 615, -1000, -1000, -1000, 104, 236, + -1000, -36, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + 208, 268, 258, 331, 33, -1000, -1000, -1000, -1000, -1000, + 70, 615, -1000, 121, -1000, -1000, 231, 10, 10, 615, + 500, -1000, 317, 466, 243, -1000, 102, 102, 118, 118, + 118, 48, 48, -1000, -1000, -1000, 10, -1000, 615, -1000, + -16, 483, -16, 186, 26, -1000, 586, 275, 249, 249, + 373, 364, 369, 65, 250, -1000, -1000, 250, -1000, 379, + 218, 218, -1000, -1000, 149, 147, 158, 153, 151, -2, + -1000, 250, -29, 250, -15, -1000, 615, 420, 10, -1000, + 615, -1000, -16, -1000, 335, 17, -1000, 10, 24, 103, + 209, 194, 134, -30, -1000, 364, -1000, 10, 10, -1000, + -1000, 376, 366, 268, 91, -1000, 130, -1000, 84, -1000, + -1000, -1000, -1000, -18, -22, -28, -1000, -1000, -1000, -1000, + 10, 615, -1000, -82, -1000, 615, 10, -1000, 306, 182, + -1000, -1000, -1000, 249, -1000, 157, 167, -1000, 386, -1000, + 373, 586, 10, 586, -1000, -1000, 209, 209, 209, 615, + -1000, 615, 282, 209, -1000, 10, 10, -1000, -1000, -1000, + 364, 65, 162, 65, 236, 236, 236, 395, -1000, 615, + -1000, 328, -35, -1000, -35, -35, 249, -1000, 393, 327, + -1000, 236, -1000, -1000, 160, -1000, 236, -1000, 236, -1000, } var yyPgo = [...]int{ - 0, 517, 513, 18, 512, 510, 508, 495, 493, 492, - 490, 489, 488, 487, 531, 486, 485, 484, 11, 27, - 482, 481, 13, 479, 478, 191, 476, 5, 17, 66, - 465, 450, 449, 14, 2, 15, 1, 447, 10, 446, - 75, 7, 445, 444, 12, 442, 441, 434, 433, 6, - 430, 3, 429, 8, 427, 425, 423, 16, 9, 20, - 214, 422, 420, 408, 407, 406, 405, 0, 403, 365, - 402, 42, 400, 165, 4, + 0, 512, 510, 23, 507, 505, 504, 502, 501, 489, + 483, 480, 479, 478, 621, 473, 472, 469, 26, 7, + 467, 466, 21, 455, 454, 17, 448, 3, 12, 5, + 447, 446, 445, 11, 2, 9, 10, 444, 18, 443, + 71, 4, 438, 437, 16, 435, 434, 432, 431, 13, + 426, 6, 424, 1, 423, 421, 419, 15, 8, 14, + 341, 242, 417, 416, 415, 410, 408, 0, 407, 355, + 406, 52, 404, 167, 19, } var yyR1 = [...]int{ @@ -435,14 +433,14 @@ var yyR1 = [...]int{ 33, 47, 47, 48, 48, 49, 49, 50, 50, 51, 52, 52, 52, 53, 53, 53, 54, 54, 54, 55, 55, 56, 56, 57, 57, 32, 32, 37, 37, 38, - 38, 58, 58, 59, 60, 60, 61, 61, 62, 62, + 38, 58, 58, 59, 61, 61, 62, 62, 60, 60, 63, 63, 63, 63, 63, 64, 64, 65, 65, 66, 66, 67, 69, 73, 74, 71, } var yyR2 = [...]int{ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 12, 3, 7, 7, 8, 7, 3, + 1, 1, 1, 12, 3, 8, 8, 8, 7, 3, 5, 8, 4, 6, 7, 4, 5, 4, 5, 5, 3, 2, 2, 2, 0, 2, 0, 2, 1, 2, 1, 1, 1, 0, 1, 1, 3, 1, 2, 3, @@ -469,48 +467,48 @@ var yyChk = [...]int{ -10, -11, -12, -13, 5, 6, 7, 8, 32, 88, 89, 91, 90, 92, 101, 102, 103, -16, 51, 52, 53, 54, -14, -72, -14, -14, -14, -14, 93, -65, - 95, 99, -62, 95, 97, 93, 93, 94, 95, 93, - -71, -71, -71, -3, 17, -17, 18, -15, 28, -25, - -69, 37, 9, -58, -59, -41, -67, -69, 37, -61, - 98, 94, -67, 37, 93, -67, -69, -60, 98, 37, - -60, -69, -18, -19, 74, -20, -69, -29, -34, -30, + 95, 99, -60, 95, 97, 93, 93, 94, 95, 93, + -71, -71, -71, -3, 17, -17, 18, -15, -60, -25, + -69, 37, 9, -58, -59, -41, -67, -69, 37, -62, + 98, 94, -67, 37, 93, -67, -69, -61, 98, 37, + -61, -69, -18, -19, 74, -20, -69, -29, -34, -30, 69, -73, -33, -41, -38, 72, 73, 80, -67, -39, -42, 20, 34, 47, 38, 39, 40, 25, -40, 98, - 83, 42, -25, 32, 82, -25, 55, 48, 82, -69, + 83, 42, 28, 32, 82, -25, 55, 48, 82, -69, 69, 37, -71, -69, -71, 96, -69, 20, 66, -67, 9, 55, -68, -67, 19, 82, 68, 67, -31, 21, 69, 23, 24, 22, 71, 70, 79, 72, 73, 74, 75, 76, 77, 78, 48, 49, 50, 43, 44, 45, 46, -29, -34, -29, -3, -36, -34, -34, -73, -34, - -34, -73, -73, -40, -73, -73, -45, -34, -55, 32, - -73, -58, -69, -28, 10, -59, -34, -67, -71, 20, - -66, 100, -63, 91, 89, 31, 90, 13, 37, -69, - -69, -71, -21, -22, -24, -73, -69, -40, -19, -67, - 74, -29, -29, -34, -35, -73, -40, 41, 21, 23, - 24, -34, -34, 25, 69, -34, -34, -34, -34, -34, - -34, -34, -34, -34, -34, -74, 104, -74, 55, -74, - -34, -74, -18, 18, -18, -33, -43, -44, 84, -32, - 35, -3, -58, -56, -41, -28, -49, 13, -29, 66, - -67, -71, -64, 96, -28, 55, -23, 56, 57, 58, - 59, 60, 62, 63, -70, -69, 19, -22, 82, -36, - -35, -34, -34, 68, 25, -34, -74, -18, -74, 55, - -46, -44, 86, -29, -57, 66, -37, -38, -57, -74, - 55, -49, -53, 15, 14, -69, -69, -47, 11, -22, - -22, 56, 61, 56, 61, 56, 56, 56, -26, 64, - 97, 65, -69, -74, -69, -74, 68, -34, -74, -33, - 87, -34, 85, 29, 55, -41, -53, -34, -50, -51, - -34, -71, -48, 12, 14, 66, 56, 56, 94, 94, - 94, -34, -74, -34, 30, -38, 55, 55, -52, 26, - 27, -49, -29, -36, -29, -73, -73, -73, 7, -34, - -51, -53, -27, -67, -27, -27, -58, -54, 16, 33, - -74, 55, -74, -74, 7, 21, -67, -67, -67, + -34, -73, -73, -40, -73, -73, -45, -34, -25, -58, + -69, -28, 10, -59, -34, -67, -71, 20, -66, 100, + -63, 91, 89, 31, 90, 13, 37, -69, -69, -71, + -21, -22, -24, -73, -69, -40, -19, -67, 74, -29, + -29, -34, -35, -73, -40, 41, 21, 23, 24, -34, + -34, 25, 69, -34, -34, -34, -34, -34, -34, -34, + -34, -34, -34, -74, 104, -74, 55, -74, -34, -74, + -18, 18, -18, -33, -43, -44, 84, -55, 32, -73, + -28, -49, 13, -29, 66, -67, -71, -64, 96, -28, + 55, -23, 56, 57, 58, 59, 60, 62, 63, -70, + -69, 19, -22, 82, -36, -35, -34, -34, 68, 25, + -34, -74, -18, -74, 55, -46, -44, 86, -29, -32, + 35, -3, -58, -56, -41, -49, -53, 15, 14, -69, + -69, -47, 11, -22, -22, 56, 61, 56, 61, 56, + 56, 56, -26, 64, 97, 65, -69, -74, -69, -74, + 68, -34, -74, -33, 87, -34, 85, -57, 66, -37, + -38, -57, -74, 55, -53, -34, -50, -51, -34, -71, + -48, 12, 14, 66, 56, 56, 94, 94, 94, -34, + -74, -34, 29, 55, -41, 55, 55, -52, 26, 27, + -49, -29, -36, -29, -73, -73, -73, 30, -38, -34, + -51, -53, -27, -67, -27, -27, 7, -54, 16, 33, + -74, 55, -74, -74, -58, 7, 21, -67, -67, -67, } var yyDef = [...]int{ 0, -2, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 34, 34, 34, 34, 34, 197, 188, 0, 0, 0, 205, 205, 205, 0, 38, 40, - 41, 42, 43, 36, 0, 0, 0, 0, 186, 0, + 41, 42, 43, 36, 188, 0, 0, 0, 186, 0, 0, 198, 0, 0, 189, 0, 184, 0, 184, 0, 31, 32, 33, 14, 39, 0, 44, 35, 0, 0, 76, 202, 0, 19, 181, 0, 145, 0, -2, 0, @@ -518,34 +516,34 @@ var yyDef = [...]int{ 0, 30, 0, 45, 47, 52, 0, 50, 51, 86, 0, 0, 115, 116, 117, 0, 0, 0, 145, 0, 135, 0, 0, 203, 147, 148, 149, 150, 180, 136, - 138, 37, 169, 0, 0, 84, 0, 0, 0, 205, + 138, 37, 0, 0, 0, 84, 0, 0, 0, 205, 0, 199, 22, 0, 25, 0, 27, 185, 0, 205, 0, 0, 48, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 103, 104, 105, 106, 107, 108, 89, 0, 0, 0, 0, 113, 128, 0, 129, - 130, 0, 0, 100, 0, 0, 0, 139, 0, 0, - 0, 84, 77, 155, 0, 182, 183, 146, 20, 187, - 0, 0, 205, 195, 190, 191, 192, 193, 194, 26, - 28, 29, 84, 55, 61, 0, 73, 75, 46, 54, - 49, 87, 88, 91, 92, 0, 110, 111, 0, 0, - 0, 94, 0, 98, 0, 118, 119, 120, 121, 122, - 123, 124, 125, 126, 127, 90, 204, 112, 0, 179, - 113, 131, 0, 0, 0, 0, 143, 140, 0, 173, - 0, 176, 173, 0, 171, 155, 163, 0, 85, 0, - 200, 23, 0, 196, 151, 0, 0, 64, 65, 0, - 0, 0, 0, 0, 78, 62, 0, 0, 0, 0, - 93, 95, 0, 0, 99, 114, 132, 0, 134, 0, - 0, 141, 0, 0, 15, 0, 175, 177, 16, 170, - 0, 163, 18, 0, 0, 205, 24, 153, 0, 56, - 59, 66, 0, 68, 0, 70, 71, 72, 57, 0, - 0, 0, 63, 58, 74, 109, 0, 96, 133, 0, - 137, 144, 0, 0, 0, 172, 17, 164, 156, 157, - 160, 21, 155, 0, 0, 0, 67, 69, 0, 0, - 0, 97, 101, 142, 0, 178, 0, 0, 159, 161, - 162, 163, 154, 152, 60, 0, 0, 0, 0, 165, - 158, 166, 0, 82, 0, 0, 174, 13, 0, 0, - 79, 0, 80, 81, 167, 0, 83, 0, 168, + 130, 0, 0, 100, 0, 0, 0, 139, 169, 84, + 77, 155, 0, 182, 183, 146, 20, 187, 0, 0, + 205, 195, 190, 191, 192, 193, 194, 26, 28, 29, + 84, 55, 61, 0, 73, 75, 46, 54, 49, 87, + 88, 91, 92, 0, 110, 111, 0, 0, 0, 94, + 0, 98, 0, 118, 119, 120, 121, 122, 123, 124, + 125, 126, 127, 90, 204, 112, 0, 179, 113, 131, + 0, 0, 0, 0, 143, 140, 0, 0, 0, 0, + 155, 163, 0, 85, 0, 200, 23, 0, 196, 151, + 0, 0, 64, 65, 0, 0, 0, 0, 0, 78, + 62, 0, 0, 0, 0, 93, 95, 0, 0, 99, + 114, 132, 0, 134, 0, 0, 141, 0, 0, 173, + 0, 176, 173, 0, 171, 163, 18, 0, 0, 205, + 24, 153, 0, 56, 59, 66, 0, 68, 0, 70, + 71, 72, 57, 0, 0, 0, 63, 58, 74, 109, + 0, 96, 133, 0, 137, 144, 0, 15, 0, 175, + 177, 16, 170, 0, 17, 164, 156, 157, 160, 21, + 155, 0, 0, 0, 67, 69, 0, 0, 0, 97, + 101, 142, 0, 0, 172, 0, 0, 159, 161, 162, + 163, 154, 152, 60, 0, 0, 0, 0, 178, 165, + 158, 166, 0, 82, 0, 0, 0, 13, 0, 0, + 79, 0, 80, 81, 174, 167, 0, 83, 0, 168, } var yyTok1 = [...]int{ @@ -920,713 +918,713 @@ yydefault: case 1: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:161 + //line sql.y:162 { setParseTree(yylex, yyDollar[1].statement) } case 2: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:167 + //line sql.y:168 { yyVAL.statement = yyDollar[1].selStmt } case 13: yyDollar = yyS[yypt-12 : yypt+1] - //line sql.y:183 + //line sql.y:184 { yyVAL.selStmt = &Select{Comments: Comments(yyDollar[2].bytes2), Distinct: yyDollar[3].str, SelectExprs: yyDollar[4].selectExprs, From: yyDollar[6].tableExprs, Where: NewWhere(AST_WHERE, yyDollar[7].boolExpr), GroupBy: GroupBy(yyDollar[8].valExprs), Having: NewWhere(AST_HAVING, yyDollar[9].boolExpr), OrderBy: yyDollar[10].orderBy, Limit: yyDollar[11].limit, Lock: yyDollar[12].str} } case 14: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:187 + //line sql.y:188 { yyVAL.selStmt = &Union{Type: yyDollar[2].str, Left: yyDollar[1].selStmt, Right: yyDollar[3].selStmt} } case 15: - yyDollar = yyS[yypt-7 : yypt+1] - //line sql.y:193 + yyDollar = yyS[yypt-8 : yypt+1] + //line sql.y:194 { - yyVAL.statement = &Insert{Comments: Comments(yyDollar[2].bytes2), Table: yyDollar[4].tableName, Columns: yyDollar[5].columns, Rows: yyDollar[6].insRows, OnDup: OnDup(yyDollar[7].updateExprs)} + yyVAL.statement = &Insert{Comments: Comments(yyDollar[2].bytes2), Ignore: yyDollar[3].str, Table: yyDollar[5].tableName, Columns: yyDollar[6].columns, Rows: yyDollar[7].insRows, OnDup: OnDup(yyDollar[8].updateExprs)} } case 16: - yyDollar = yyS[yypt-7 : yypt+1] - //line sql.y:197 + yyDollar = yyS[yypt-8 : yypt+1] + //line sql.y:198 { - cols := make(Columns, 0, len(yyDollar[6].updateExprs)) - vals := make(ValTuple, 0, len(yyDollar[6].updateExprs)) - for _, col := range yyDollar[6].updateExprs { + cols := make(Columns, 0, len(yyDollar[7].updateExprs)) + vals := make(ValTuple, 0, len(yyDollar[7].updateExprs)) + for _, col := range yyDollar[7].updateExprs { cols = append(cols, &NonStarExpr{Expr: col.Name}) vals = append(vals, col.Expr) } - yyVAL.statement = &Insert{Comments: Comments(yyDollar[2].bytes2), Table: yyDollar[4].tableName, Columns: cols, Rows: Values{vals}, OnDup: OnDup(yyDollar[7].updateExprs)} + yyVAL.statement = &Insert{Comments: Comments(yyDollar[2].bytes2), Ignore: yyDollar[3].str, Table: yyDollar[5].tableName, Columns: cols, Rows: Values{vals}, OnDup: OnDup(yyDollar[8].updateExprs)} } case 17: yyDollar = yyS[yypt-8 : yypt+1] - //line sql.y:209 + //line sql.y:210 { yyVAL.statement = &Update{Comments: Comments(yyDollar[2].bytes2), Table: yyDollar[3].tableName, Exprs: yyDollar[5].updateExprs, Where: NewWhere(AST_WHERE, yyDollar[6].boolExpr), OrderBy: yyDollar[7].orderBy, Limit: yyDollar[8].limit} } case 18: yyDollar = yyS[yypt-7 : yypt+1] - //line sql.y:215 + //line sql.y:216 { yyVAL.statement = &Delete{Comments: Comments(yyDollar[2].bytes2), Table: yyDollar[4].tableName, Where: NewWhere(AST_WHERE, yyDollar[5].boolExpr), OrderBy: yyDollar[6].orderBy, Limit: yyDollar[7].limit} } case 19: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:221 + //line sql.y:222 { yyVAL.statement = &Set{Comments: Comments(yyDollar[2].bytes2), Exprs: yyDollar[3].updateExprs} } case 20: yyDollar = yyS[yypt-5 : yypt+1] - //line sql.y:227 + //line sql.y:228 { yyVAL.statement = &DDL{Action: AST_CREATE, NewName: yyDollar[4].sqlID} } case 21: yyDollar = yyS[yypt-8 : yypt+1] - //line sql.y:231 + //line sql.y:232 { // Change this to an alter statement yyVAL.statement = &DDL{Action: AST_ALTER, Table: yyDollar[7].sqlID, NewName: yyDollar[7].sqlID} } case 22: yyDollar = yyS[yypt-4 : yypt+1] - //line sql.y:236 + //line sql.y:237 { yyVAL.statement = &DDL{Action: AST_CREATE, NewName: SQLName(yyDollar[3].sqlID)} } case 23: yyDollar = yyS[yypt-6 : yypt+1] - //line sql.y:242 + //line sql.y:243 { yyVAL.statement = &DDL{Action: AST_ALTER, Table: yyDollar[4].sqlID, NewName: yyDollar[4].sqlID} } case 24: yyDollar = yyS[yypt-7 : yypt+1] - //line sql.y:246 + //line sql.y:247 { // Change this to a rename statement yyVAL.statement = &DDL{Action: AST_RENAME, Table: yyDollar[4].sqlID, NewName: yyDollar[7].sqlID} } case 25: yyDollar = yyS[yypt-4 : yypt+1] - //line sql.y:251 + //line sql.y:252 { yyVAL.statement = &DDL{Action: AST_ALTER, Table: SQLName(yyDollar[3].sqlID), NewName: SQLName(yyDollar[3].sqlID)} } case 26: yyDollar = yyS[yypt-5 : yypt+1] - //line sql.y:257 + //line sql.y:258 { yyVAL.statement = &DDL{Action: AST_RENAME, Table: yyDollar[3].sqlID, NewName: yyDollar[5].sqlID} } case 27: yyDollar = yyS[yypt-4 : yypt+1] - //line sql.y:263 + //line sql.y:264 { yyVAL.statement = &DDL{Action: AST_DROP, Table: yyDollar[4].sqlID} } case 28: yyDollar = yyS[yypt-5 : yypt+1] - //line sql.y:267 + //line sql.y:268 { // Change this to an alter statement yyVAL.statement = &DDL{Action: AST_ALTER, Table: yyDollar[5].sqlID, NewName: yyDollar[5].sqlID} } case 29: yyDollar = yyS[yypt-5 : yypt+1] - //line sql.y:272 + //line sql.y:273 { yyVAL.statement = &DDL{Action: AST_DROP, Table: SQLName(yyDollar[4].sqlID)} } case 30: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:278 + //line sql.y:279 { yyVAL.statement = &DDL{Action: AST_ALTER, Table: yyDollar[3].sqlID, NewName: yyDollar[3].sqlID} } case 31: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:284 + //line sql.y:285 { yyVAL.statement = &Other{} } case 32: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:288 + //line sql.y:289 { yyVAL.statement = &Other{} } case 33: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:292 + //line sql.y:293 { yyVAL.statement = &Other{} } case 34: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:297 + //line sql.y:298 { setAllowComments(yylex, true) } case 35: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:301 + //line sql.y:302 { yyVAL.bytes2 = yyDollar[2].bytes2 setAllowComments(yylex, false) } case 36: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:307 + //line sql.y:308 { yyVAL.bytes2 = nil } case 37: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:311 + //line sql.y:312 { yyVAL.bytes2 = append(yyDollar[1].bytes2, yyDollar[2].bytes) } case 38: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:317 + //line sql.y:318 { yyVAL.str = AST_UNION } case 39: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:321 + //line sql.y:322 { yyVAL.str = AST_UNION_ALL } case 40: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:325 + //line sql.y:326 { yyVAL.str = AST_SET_MINUS } case 41: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:329 + //line sql.y:330 { yyVAL.str = AST_EXCEPT } case 42: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:333 + //line sql.y:334 { yyVAL.str = AST_INTERSECT } case 43: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:338 + //line sql.y:339 { yyVAL.str = "" } case 44: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:342 + //line sql.y:343 { yyVAL.str = AST_DISTINCT } case 45: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:348 + //line sql.y:349 { yyVAL.selectExprs = SelectExprs{yyDollar[1].selectExpr} } case 46: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:352 + //line sql.y:353 { yyVAL.selectExprs = append(yyVAL.selectExprs, yyDollar[3].selectExpr) } case 47: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:358 + //line sql.y:359 { yyVAL.selectExpr = &StarExpr{} } case 48: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:362 + //line sql.y:363 { yyVAL.selectExpr = &NonStarExpr{Expr: yyDollar[1].expr, As: yyDollar[2].sqlID} } case 49: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:366 + //line sql.y:367 { yyVAL.selectExpr = &StarExpr{TableName: yyDollar[1].sqlID} } case 50: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:372 + //line sql.y:373 { yyVAL.expr = yyDollar[1].boolExpr } case 51: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:376 + //line sql.y:377 { yyVAL.expr = yyDollar[1].valExpr } case 52: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:381 + //line sql.y:382 { yyVAL.sqlID = "" } case 53: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:385 + //line sql.y:386 { yyVAL.sqlID = yyDollar[1].sqlID } case 54: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:389 + //line sql.y:390 { yyVAL.sqlID = yyDollar[2].sqlID } case 55: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:395 + //line sql.y:396 { yyVAL.tableExprs = TableExprs{yyDollar[1].tableExpr} } case 56: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:399 + //line sql.y:400 { yyVAL.tableExprs = append(yyVAL.tableExprs, yyDollar[3].tableExpr) } case 57: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:405 + //line sql.y:406 { yyVAL.tableExpr = &AliasedTableExpr{Expr: yyDollar[1].smTableExpr, As: yyDollar[2].sqlID, Hints: yyDollar[3].indexHints} } case 58: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:409 + //line sql.y:410 { yyVAL.tableExpr = &ParenTableExpr{Expr: yyDollar[2].tableExpr} } case 59: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:413 + //line sql.y:414 { yyVAL.tableExpr = &JoinTableExpr{LeftExpr: yyDollar[1].tableExpr, Join: yyDollar[2].str, RightExpr: yyDollar[3].tableExpr} } case 60: yyDollar = yyS[yypt-5 : yypt+1] - //line sql.y:417 + //line sql.y:418 { yyVAL.tableExpr = &JoinTableExpr{LeftExpr: yyDollar[1].tableExpr, Join: yyDollar[2].str, RightExpr: yyDollar[3].tableExpr, On: yyDollar[5].boolExpr} } case 61: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:422 + //line sql.y:423 { yyVAL.sqlID = "" } case 62: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:426 + //line sql.y:427 { yyVAL.sqlID = yyDollar[1].sqlID } case 63: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:430 + //line sql.y:431 { yyVAL.sqlID = yyDollar[2].sqlID } case 64: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:436 + //line sql.y:437 { yyVAL.str = AST_JOIN } case 65: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:440 + //line sql.y:441 { yyVAL.str = AST_STRAIGHT_JOIN } case 66: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:444 + //line sql.y:445 { yyVAL.str = AST_LEFT_JOIN } case 67: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:448 + //line sql.y:449 { yyVAL.str = AST_LEFT_JOIN } case 68: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:452 + //line sql.y:453 { yyVAL.str = AST_RIGHT_JOIN } case 69: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:456 + //line sql.y:457 { yyVAL.str = AST_RIGHT_JOIN } case 70: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:460 + //line sql.y:461 { yyVAL.str = AST_JOIN } case 71: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:464 + //line sql.y:465 { yyVAL.str = AST_CROSS_JOIN } case 72: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:468 + //line sql.y:469 { yyVAL.str = AST_NATURAL_JOIN } case 73: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:474 + //line sql.y:475 { yyVAL.smTableExpr = &TableName{Name: yyDollar[1].sqlID} } case 74: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:478 + //line sql.y:479 { yyVAL.smTableExpr = &TableName{Qualifier: yyDollar[1].sqlID, Name: yyDollar[3].sqlID} } case 75: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:482 + //line sql.y:483 { yyVAL.smTableExpr = yyDollar[1].subquery } case 76: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:488 + //line sql.y:489 { yyVAL.tableName = &TableName{Name: yyDollar[1].sqlID} } case 77: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:492 + //line sql.y:493 { yyVAL.tableName = &TableName{Qualifier: yyDollar[1].sqlID, Name: yyDollar[3].sqlID} } case 78: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:497 + //line sql.y:498 { yyVAL.indexHints = nil } case 79: yyDollar = yyS[yypt-5 : yypt+1] - //line sql.y:501 + //line sql.y:502 { yyVAL.indexHints = &IndexHints{Type: AST_USE, Indexes: yyDollar[4].sqlIDs} } case 80: yyDollar = yyS[yypt-5 : yypt+1] - //line sql.y:505 + //line sql.y:506 { yyVAL.indexHints = &IndexHints{Type: AST_IGNORE, Indexes: yyDollar[4].sqlIDs} } case 81: yyDollar = yyS[yypt-5 : yypt+1] - //line sql.y:509 + //line sql.y:510 { yyVAL.indexHints = &IndexHints{Type: AST_FORCE, Indexes: yyDollar[4].sqlIDs} } case 82: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:515 + //line sql.y:516 { yyVAL.sqlIDs = []SQLName{yyDollar[1].sqlID} } case 83: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:519 + //line sql.y:520 { yyVAL.sqlIDs = append(yyDollar[1].sqlIDs, yyDollar[3].sqlID) } case 84: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:524 + //line sql.y:525 { yyVAL.boolExpr = nil } case 85: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:528 + //line sql.y:529 { yyVAL.boolExpr = yyDollar[2].boolExpr } case 87: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:535 + //line sql.y:536 { yyVAL.boolExpr = &AndExpr{Left: yyDollar[1].boolExpr, Right: yyDollar[3].boolExpr} } case 88: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:539 + //line sql.y:540 { yyVAL.boolExpr = &OrExpr{Left: yyDollar[1].boolExpr, Right: yyDollar[3].boolExpr} } case 89: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:543 + //line sql.y:544 { yyVAL.boolExpr = &NotExpr{Expr: yyDollar[2].boolExpr} } case 90: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:547 + //line sql.y:548 { yyVAL.boolExpr = &ParenBoolExpr{Expr: yyDollar[2].boolExpr} } case 91: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:553 + //line sql.y:554 { yyVAL.boolExpr = &ComparisonExpr{Left: yyDollar[1].valExpr, Operator: yyDollar[2].str, Right: yyDollar[3].valExpr} } case 92: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:557 + //line sql.y:558 { yyVAL.boolExpr = &ComparisonExpr{Left: yyDollar[1].valExpr, Operator: AST_IN, Right: yyDollar[3].colTuple} } case 93: yyDollar = yyS[yypt-4 : yypt+1] - //line sql.y:561 + //line sql.y:562 { yyVAL.boolExpr = &ComparisonExpr{Left: yyDollar[1].valExpr, Operator: AST_NOT_IN, Right: yyDollar[4].colTuple} } case 94: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:565 + //line sql.y:566 { yyVAL.boolExpr = &ComparisonExpr{Left: yyDollar[1].valExpr, Operator: AST_LIKE, Right: yyDollar[3].valExpr} } case 95: yyDollar = yyS[yypt-4 : yypt+1] - //line sql.y:569 + //line sql.y:570 { yyVAL.boolExpr = &ComparisonExpr{Left: yyDollar[1].valExpr, Operator: AST_NOT_LIKE, Right: yyDollar[4].valExpr} } case 96: yyDollar = yyS[yypt-5 : yypt+1] - //line sql.y:573 + //line sql.y:574 { yyVAL.boolExpr = &RangeCond{Left: yyDollar[1].valExpr, Operator: AST_BETWEEN, From: yyDollar[3].valExpr, To: yyDollar[5].valExpr} } case 97: yyDollar = yyS[yypt-6 : yypt+1] - //line sql.y:577 + //line sql.y:578 { yyVAL.boolExpr = &RangeCond{Left: yyDollar[1].valExpr, Operator: AST_NOT_BETWEEN, From: yyDollar[4].valExpr, To: yyDollar[6].valExpr} } case 98: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:581 + //line sql.y:582 { yyVAL.boolExpr = &NullCheck{Operator: AST_IS_NULL, Expr: yyDollar[1].valExpr} } case 99: yyDollar = yyS[yypt-4 : yypt+1] - //line sql.y:585 + //line sql.y:586 { yyVAL.boolExpr = &NullCheck{Operator: AST_IS_NOT_NULL, Expr: yyDollar[1].valExpr} } case 100: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:589 + //line sql.y:590 { yyVAL.boolExpr = &ExistsExpr{Subquery: yyDollar[2].subquery} } case 101: yyDollar = yyS[yypt-6 : yypt+1] - //line sql.y:593 + //line sql.y:594 { yyVAL.boolExpr = &KeyrangeExpr{Start: yyDollar[3].valExpr, End: yyDollar[5].valExpr} } case 102: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:599 + //line sql.y:600 { yyVAL.str = AST_EQ } case 103: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:603 + //line sql.y:604 { yyVAL.str = AST_LT } case 104: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:607 + //line sql.y:608 { yyVAL.str = AST_GT } case 105: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:611 + //line sql.y:612 { yyVAL.str = AST_LE } case 106: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:615 + //line sql.y:616 { yyVAL.str = AST_GE } case 107: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:619 + //line sql.y:620 { yyVAL.str = AST_NE } case 108: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:623 + //line sql.y:624 { yyVAL.str = AST_NSE } case 109: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:629 + //line sql.y:630 { yyVAL.colTuple = ValTuple(yyDollar[2].valExprs) } case 110: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:633 + //line sql.y:634 { yyVAL.colTuple = yyDollar[1].subquery } case 111: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:637 + //line sql.y:638 { yyVAL.colTuple = ListArg(yyDollar[1].bytes) } case 112: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:643 + //line sql.y:644 { yyVAL.subquery = &Subquery{yyDollar[2].selStmt} } case 113: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:649 + //line sql.y:650 { yyVAL.valExprs = ValExprs{yyDollar[1].valExpr} } case 114: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:653 + //line sql.y:654 { yyVAL.valExprs = append(yyDollar[1].valExprs, yyDollar[3].valExpr) } case 115: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:659 + //line sql.y:660 { yyVAL.valExpr = yyDollar[1].valExpr } case 116: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:663 + //line sql.y:664 { yyVAL.valExpr = yyDollar[1].colName } case 117: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:667 + //line sql.y:668 { yyVAL.valExpr = yyDollar[1].rowTuple } case 118: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:671 + //line sql.y:672 { yyVAL.valExpr = &BinaryExpr{Left: yyDollar[1].valExpr, Operator: AST_BITAND, Right: yyDollar[3].valExpr} } case 119: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:675 + //line sql.y:676 { yyVAL.valExpr = &BinaryExpr{Left: yyDollar[1].valExpr, Operator: AST_BITOR, Right: yyDollar[3].valExpr} } case 120: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:679 + //line sql.y:680 { yyVAL.valExpr = &BinaryExpr{Left: yyDollar[1].valExpr, Operator: AST_BITXOR, Right: yyDollar[3].valExpr} } case 121: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:683 + //line sql.y:684 { yyVAL.valExpr = &BinaryExpr{Left: yyDollar[1].valExpr, Operator: AST_PLUS, Right: yyDollar[3].valExpr} } case 122: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:687 + //line sql.y:688 { yyVAL.valExpr = &BinaryExpr{Left: yyDollar[1].valExpr, Operator: AST_MINUS, Right: yyDollar[3].valExpr} } case 123: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:691 + //line sql.y:692 { yyVAL.valExpr = &BinaryExpr{Left: yyDollar[1].valExpr, Operator: AST_MULT, Right: yyDollar[3].valExpr} } case 124: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:695 + //line sql.y:696 { yyVAL.valExpr = &BinaryExpr{Left: yyDollar[1].valExpr, Operator: AST_DIV, Right: yyDollar[3].valExpr} } case 125: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:699 + //line sql.y:700 { yyVAL.valExpr = &BinaryExpr{Left: yyDollar[1].valExpr, Operator: AST_MOD, Right: yyDollar[3].valExpr} } case 126: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:703 + //line sql.y:704 { yyVAL.valExpr = &BinaryExpr{Left: yyDollar[1].valExpr, Operator: AST_SHIFT_LEFT, Right: yyDollar[3].valExpr} } case 127: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:707 + //line sql.y:708 { yyVAL.valExpr = &BinaryExpr{Left: yyDollar[1].valExpr, Operator: AST_SHIFT_RIGHT, Right: yyDollar[3].valExpr} } case 128: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:711 + //line sql.y:712 { if num, ok := yyDollar[2].valExpr.(NumVal); ok { yyVAL.valExpr = num @@ -1636,7 +1634,7 @@ yydefault: } case 129: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:719 + //line sql.y:720 { if num, ok := yyDollar[2].valExpr.(NumVal); ok { // Handle double negative @@ -1651,235 +1649,235 @@ yydefault: } case 130: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:732 + //line sql.y:733 { yyVAL.valExpr = &UnaryExpr{Operator: AST_TILDA, Expr: yyDollar[2].valExpr} } case 131: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:736 + //line sql.y:737 { yyVAL.valExpr = &FuncExpr{Name: string(yyDollar[1].sqlID)} } case 132: yyDollar = yyS[yypt-4 : yypt+1] - //line sql.y:740 + //line sql.y:741 { yyVAL.valExpr = &FuncExpr{Name: string(yyDollar[1].sqlID), Exprs: yyDollar[3].selectExprs} } case 133: yyDollar = yyS[yypt-5 : yypt+1] - //line sql.y:744 + //line sql.y:745 { yyVAL.valExpr = &FuncExpr{Name: string(yyDollar[1].sqlID), Distinct: true, Exprs: yyDollar[4].selectExprs} } case 134: yyDollar = yyS[yypt-4 : yypt+1] - //line sql.y:748 + //line sql.y:749 { yyVAL.valExpr = &FuncExpr{Name: yyDollar[1].str, Exprs: yyDollar[3].selectExprs} } case 135: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:752 + //line sql.y:753 { yyVAL.valExpr = yyDollar[1].caseExpr } case 136: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:758 + //line sql.y:759 { yyVAL.str = "if" } case 137: yyDollar = yyS[yypt-5 : yypt+1] - //line sql.y:764 + //line sql.y:765 { yyVAL.caseExpr = &CaseExpr{Expr: yyDollar[2].valExpr, Whens: yyDollar[3].whens, Else: yyDollar[4].valExpr} } case 138: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:769 + //line sql.y:770 { yyVAL.valExpr = nil } case 139: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:773 + //line sql.y:774 { yyVAL.valExpr = yyDollar[1].valExpr } case 140: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:779 + //line sql.y:780 { yyVAL.whens = []*When{yyDollar[1].when} } case 141: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:783 + //line sql.y:784 { yyVAL.whens = append(yyDollar[1].whens, yyDollar[2].when) } case 142: yyDollar = yyS[yypt-4 : yypt+1] - //line sql.y:789 + //line sql.y:790 { yyVAL.when = &When{Cond: yyDollar[2].boolExpr, Val: yyDollar[4].valExpr} } case 143: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:794 + //line sql.y:795 { yyVAL.valExpr = nil } case 144: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:798 + //line sql.y:799 { yyVAL.valExpr = yyDollar[2].valExpr } case 145: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:804 + //line sql.y:805 { yyVAL.colName = &ColName{Name: yyDollar[1].sqlID} } case 146: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:808 + //line sql.y:809 { yyVAL.colName = &ColName{Qualifier: yyDollar[1].sqlID, Name: yyDollar[3].sqlID} } case 147: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:814 + //line sql.y:815 { yyVAL.valExpr = StrVal(yyDollar[1].bytes) } case 148: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:818 + //line sql.y:819 { yyVAL.valExpr = NumVal(yyDollar[1].bytes) } case 149: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:822 + //line sql.y:823 { yyVAL.valExpr = ValArg(yyDollar[1].bytes) } case 150: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:826 + //line sql.y:827 { yyVAL.valExpr = &NullVal{} } case 151: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:831 + //line sql.y:832 { yyVAL.valExprs = nil } case 152: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:835 + //line sql.y:836 { yyVAL.valExprs = yyDollar[3].valExprs } case 153: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:840 + //line sql.y:841 { yyVAL.boolExpr = nil } case 154: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:844 + //line sql.y:845 { yyVAL.boolExpr = yyDollar[2].boolExpr } case 155: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:849 + //line sql.y:850 { yyVAL.orderBy = nil } case 156: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:853 + //line sql.y:854 { yyVAL.orderBy = yyDollar[3].orderBy } case 157: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:859 + //line sql.y:860 { yyVAL.orderBy = OrderBy{yyDollar[1].order} } case 158: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:863 + //line sql.y:864 { yyVAL.orderBy = append(yyDollar[1].orderBy, yyDollar[3].order) } case 159: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:869 + //line sql.y:870 { yyVAL.order = &Order{Expr: yyDollar[1].valExpr, Direction: yyDollar[2].str} } case 160: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:874 + //line sql.y:875 { yyVAL.str = AST_ASC } case 161: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:878 + //line sql.y:879 { yyVAL.str = AST_ASC } case 162: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:882 + //line sql.y:883 { yyVAL.str = AST_DESC } case 163: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:887 + //line sql.y:888 { yyVAL.limit = nil } case 164: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:891 + //line sql.y:892 { yyVAL.limit = &Limit{Rowcount: yyDollar[2].valExpr} } case 165: yyDollar = yyS[yypt-4 : yypt+1] - //line sql.y:895 + //line sql.y:896 { yyVAL.limit = &Limit{Offset: yyDollar[2].valExpr, Rowcount: yyDollar[4].valExpr} } case 166: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:900 + //line sql.y:901 { yyVAL.str = "" } case 167: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:904 + //line sql.y:905 { yyVAL.str = AST_FOR_UPDATE } case 168: yyDollar = yyS[yypt-4 : yypt+1] - //line sql.y:908 + //line sql.y:909 { if yyDollar[3].sqlID != "share" { yylex.Error("expecting share") @@ -1893,211 +1891,211 @@ yydefault: } case 169: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:921 + //line sql.y:922 { yyVAL.columns = nil } case 170: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:925 + //line sql.y:926 { yyVAL.columns = yyDollar[2].columns } case 171: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:931 + //line sql.y:932 { yyVAL.columns = Columns{&NonStarExpr{Expr: yyDollar[1].colName}} } case 172: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:935 + //line sql.y:936 { yyVAL.columns = append(yyVAL.columns, &NonStarExpr{Expr: yyDollar[3].colName}) } case 173: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:940 + //line sql.y:941 { yyVAL.updateExprs = nil } case 174: yyDollar = yyS[yypt-5 : yypt+1] - //line sql.y:944 + //line sql.y:945 { yyVAL.updateExprs = yyDollar[5].updateExprs } case 175: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:950 + //line sql.y:951 { yyVAL.insRows = yyDollar[2].values } case 176: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:954 + //line sql.y:955 { yyVAL.insRows = yyDollar[1].selStmt } case 177: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:960 + //line sql.y:961 { yyVAL.values = Values{yyDollar[1].rowTuple} } case 178: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:964 + //line sql.y:965 { yyVAL.values = append(yyDollar[1].values, yyDollar[3].rowTuple) } case 179: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:970 + //line sql.y:971 { yyVAL.rowTuple = ValTuple(yyDollar[2].valExprs) } case 180: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:974 + //line sql.y:975 { yyVAL.rowTuple = yyDollar[1].subquery } case 181: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:980 + //line sql.y:981 { yyVAL.updateExprs = UpdateExprs{yyDollar[1].updateExpr} } case 182: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:984 + //line sql.y:985 { yyVAL.updateExprs = append(yyDollar[1].updateExprs, yyDollar[3].updateExpr) } case 183: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:990 + //line sql.y:991 { yyVAL.updateExpr = &UpdateExpr{Name: yyDollar[1].colName, Expr: yyDollar[3].valExpr} } case 184: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:995 + //line sql.y:996 { yyVAL.empty = struct{}{} } case 185: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:997 + //line sql.y:998 { yyVAL.empty = struct{}{} } case 186: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:1000 + //line sql.y:1001 { yyVAL.empty = struct{}{} } case 187: yyDollar = yyS[yypt-3 : yypt+1] - //line sql.y:1002 + //line sql.y:1003 { yyVAL.empty = struct{}{} } case 188: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:1005 + //line sql.y:1006 { - yyVAL.empty = struct{}{} + yyVAL.str = "" } case 189: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:1007 + //line sql.y:1008 { - yyVAL.empty = struct{}{} + yyVAL.str = AST_IGNORE } case 190: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:1011 + //line sql.y:1012 { yyVAL.empty = struct{}{} } case 191: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:1013 + //line sql.y:1014 { yyVAL.empty = struct{}{} } case 192: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:1015 + //line sql.y:1016 { yyVAL.empty = struct{}{} } case 193: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:1017 + //line sql.y:1018 { yyVAL.empty = struct{}{} } case 194: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:1019 + //line sql.y:1020 { yyVAL.empty = struct{}{} } case 195: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:1022 + //line sql.y:1023 { yyVAL.empty = struct{}{} } case 196: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:1024 + //line sql.y:1025 { yyVAL.empty = struct{}{} } case 197: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:1027 + //line sql.y:1028 { yyVAL.empty = struct{}{} } case 198: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:1029 + //line sql.y:1030 { yyVAL.empty = struct{}{} } case 199: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:1032 + //line sql.y:1033 { yyVAL.empty = struct{}{} } case 200: yyDollar = yyS[yypt-2 : yypt+1] - //line sql.y:1034 + //line sql.y:1035 { yyVAL.empty = struct{}{} } case 201: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:1038 + //line sql.y:1039 { yyVAL.sqlID = SQLName(strings.ToLower(string(yyDollar[1].bytes))) } case 202: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:1044 + //line sql.y:1045 { yyVAL.sqlID = SQLName(yyDollar[1].bytes) } case 203: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:1050 + //line sql.y:1051 { if incNesting(yylex) { yylex.Error("max nesting level reached") @@ -2106,13 +2104,13 @@ yydefault: } case 204: yyDollar = yyS[yypt-1 : yypt+1] - //line sql.y:1059 + //line sql.y:1060 { decNesting(yylex) } case 205: yyDollar = yyS[yypt-0 : yypt+1] - //line sql.y:1064 + //line sql.y:1065 { forceEOF(yylex) } diff --git a/go/vt/sqlparser/sql.y b/go/vt/sqlparser/sql.y index 14f7393f85..cb825077ae 100644 --- a/go/vt/sqlparser/sql.y +++ b/go/vt/sqlparser/sql.y @@ -149,7 +149,8 @@ func forceEOF(yylex interface{}) { %type on_dup_opt %type update_list %type update_expression -%type exists_opt not_exists_opt ignore_opt non_rename_operation to_opt constraint_opt using_opt +%type ignore_opt +%type exists_opt not_exists_opt non_rename_operation to_opt constraint_opt using_opt %type sql_id as_lower_opt %type table_id as_opt %type force_eof @@ -189,19 +190,19 @@ select_statement: } insert_statement: - INSERT comment_opt INTO dml_table_expression column_list_opt row_list on_dup_opt + INSERT comment_opt ignore_opt INTO dml_table_expression column_list_opt row_list on_dup_opt { - $$ = &Insert{Comments: Comments($2), Table: $4, Columns: $5, Rows: $6, OnDup: OnDup($7)} + $$ = &Insert{Comments: Comments($2), Ignore: $3, Table: $5, Columns: $6, Rows: $7, OnDup: OnDup($8)} } -| INSERT comment_opt INTO dml_table_expression SET update_list on_dup_opt +| INSERT comment_opt ignore_opt INTO dml_table_expression SET update_list on_dup_opt { - cols := make(Columns, 0, len($6)) - vals := make(ValTuple, 0, len($6)) - for _, col := range $6 { + cols := make(Columns, 0, len($7)) + vals := make(ValTuple, 0, len($7)) + for _, col := range $7 { cols = append(cols, &NonStarExpr{Expr: col.Name}) vals = append(vals, col.Expr) } - $$ = &Insert{Comments: Comments($2), Table: $4, Columns: cols, Rows: Values{vals}, OnDup: OnDup($7)} + $$ = &Insert{Comments: Comments($2), Ignore: $3, Table: $5, Columns: cols, Rows: Values{vals}, OnDup: OnDup($8)} } update_statement: @@ -1002,9 +1003,9 @@ not_exists_opt: { $$ = struct{}{} } ignore_opt: - { $$ = struct{}{} } + { $$ = "" } | IGNORE - { $$ = struct{}{} } + { $$ = AST_IGNORE } non_rename_operation: ALTER diff --git a/go/vt/tabletserver/planbuilder/dml.go b/go/vt/tabletserver/planbuilder/dml.go index 8a8ef808c3..1e55410513 100644 --- a/go/vt/tabletserver/planbuilder/dml.go +++ b/go/vt/tabletserver/planbuilder/dml.go @@ -425,10 +425,10 @@ func analyzeInsert(ins *sqlparser.Insert, getTable TableGetter) (plan *ExecPlan, plan.Reason = REASON_COMPLEX_EXPR return plan, nil } - plan.OuterQuery = GenerateInsertNoUpdate(ins) plan.PKValues = pkValues if ins.OnDup == nil { plan.PlanId = PLAN_INSERT_PK + plan.OuterQuery = sqlparser.GenerateParsedQuery(ins) return plan, nil } if len(rowList) > 1 { @@ -445,6 +445,10 @@ func analyzeInsert(ins *sqlparser.Insert, getTable TableGetter) (plan *ExecPlan, return nil, err } plan.PlanId = PLAN_UPSERT_PK + newins := *ins + newins.Ignore = "" + newins.OnDup = nil + plan.OuterQuery = sqlparser.GenerateParsedQuery(&newins) upd := &sqlparser.Update{ Comments: ins.Comments, Table: ins.Table, diff --git a/go/vt/tabletserver/planbuilder/query_gen.go b/go/vt/tabletserver/planbuilder/query_gen.go index b96144e74a..77c8f8616e 100644 --- a/go/vt/tabletserver/planbuilder/query_gen.go +++ b/go/vt/tabletserver/planbuilder/query_gen.go @@ -78,8 +78,9 @@ func GenerateSelectOuterQuery(sel *sqlparser.Select, tableInfo *schema.Table) *s // GenerateInsertOuterQuery generates the outer query for inserts. func GenerateInsertOuterQuery(ins *sqlparser.Insert) *sqlparser.ParsedQuery { buf := sqlparser.NewTrackedBuffer(nil) - buf.Myprintf("insert %vinto %v%v values %a", + buf.Myprintf("insert %v%sinto %v%v values %a", ins.Comments, + ins.Ignore, ins.Table, ins.Columns, ":#values", @@ -87,15 +88,6 @@ func GenerateInsertOuterQuery(ins *sqlparser.Insert) *sqlparser.ParsedQuery { return buf.ParsedQuery() } -// GenerateInsertNoUpdate generates an insert without the update part. -func GenerateInsertNoUpdate(ins *sqlparser.Insert) *sqlparser.ParsedQuery { - buf := sqlparser.NewTrackedBuffer(nil) - buf.Myprintf("insert %vinto %v%v %v", - ins.Comments, - ins.Table, ins.Columns, ins.Rows) - return buf.ParsedQuery() -} - // GenerateUpdateOuterQuery generates the outer query for updates. func GenerateUpdateOuterQuery(upd *sqlparser.Update) *sqlparser.ParsedQuery { buf := sqlparser.NewTrackedBuffer(nil) diff --git a/test/queryservice_tests/nocache_cases.py b/test/queryservice_tests/nocache_cases.py index ccc3e112ac..0cc5e5a345 100644 --- a/test/queryservice_tests/nocache_cases.py +++ b/test/queryservice_tests/nocache_cases.py @@ -265,6 +265,26 @@ cases = [ 'delete from vtocc_a where eid>1', 'commit']), + MultiCase( + 'insert ignore', + ['begin', + Case(sql="insert /* simple */ ignore into vtocc_a values (2, 1, 'aaaa', 'bbbb')", + rewritten="insert /* simple */ ignore into vtocc_a values (2, 1, 'aaaa', 'bbbb') /* _stream vtocc_a (eid id ) (2 1 )", + rowcount=1), + 'commit', + Case(sql='select * from vtocc_a where eid = 2 and id = 1', + result=[(2L, 1L, 'aaaa', 'bbbb')]), + 'begin', + Case(sql="insert /* simple */ ignore into vtocc_a values (2, 1, 'cccc', 'cccc')", + rewritten="insert /* simple */ ignore into vtocc_a values (2, 1, 'cccc', 'cccc') /* _stream vtocc_a (eid id ) (2 1 )", + rowcount=0), + 'commit', + Case(sql='select * from vtocc_a where eid = 2 and id = 1', + result=[(2L, 1L, 'aaaa', 'bbbb')]), + 'begin', + 'delete from vtocc_a where eid>1', + 'commit']), + MultiCase( 'qualified insert', ['begin', @@ -431,8 +451,13 @@ cases = [ "insert into upsert_test(id1, id2) values (1, 2) /* _stream upsert_test (id1 ) (1 )", "update upsert_test set id2 = 2 where id1 in (1) /* _stream upsert_test (id1 ) (1 )"], rowcount=0), + Case(sql="insert ignore into upsert_test(id1, id2) values (1, 3) on duplicate key update id2 = 3", + rewritten=[ + "insert into upsert_test(id1, id2) values (1, 3) /* _stream upsert_test (id1 ) (1 )", + "update upsert_test set id2 = 3 where id1 in (1) /* _stream upsert_test (id1 ) (1 )"], + rowcount=2), Case(sql='select * from upsert_test', - result=[(1L, 2L)]), + result=[(1L, 3L)]), 'commit', 'begin', 'delete from upsert_test', From 7ce29af57218774b903c9a0e0ebef0fb19c698a6 Mon Sep 17 00:00:00 2001 From: Michael Berlin Date: Tue, 18 Aug 2015 13:23:22 -0700 Subject: [PATCH 41/80] tabletmanager: Return filtered replication stats as multiple values instead using an additional struct. --- go/vt/tabletmanager/binlog.go | 20 +++++--------------- go/vt/tabletmanager/healthcheck.go | 4 +--- 2 files changed, 6 insertions(+), 18 deletions(-) diff --git a/go/vt/tabletmanager/binlog.go b/go/vt/tabletmanager/binlog.go index cd6df4652b..98b24af866 100644 --- a/go/vt/tabletmanager/binlog.go +++ b/go/vt/tabletmanager/binlog.go @@ -679,23 +679,13 @@ func (blm *BinlogPlayerMap) Status() *BinlogPlayerMapStatus { return result } -// BinlogPlayerMapStatusSummary contains aggregated health information e.g. +// StatusSummary returns aggregated health information e.g. // the maximum replication delay across all binlog players. -type BinlogPlayerMapStatusSummary struct { - // TODO(mberlin): Record here LastError and State != Running as well? - maxSecondsBehindMaster int64 - binlogPlayersCount int32 -} - -// StatusSummary returns a summary of the state of all active binlog players. // It is used by the QueryService.StreamHealth RPC. -func (blm *BinlogPlayerMap) StatusSummary() *BinlogPlayerMapStatusSummary { - result := &BinlogPlayerMapStatusSummary{} - +func (blm *BinlogPlayerMap) StatusSummary() (maxSecondsBehindMaster int64, binlogPlayersCount int32) { blm.mu.Lock() defer blm.mu.Unlock() - result.maxSecondsBehindMaster = blm.maxSecondsBehindMaster_UNGUARDED() - result.binlogPlayersCount = int32(len(blm.players)) - - return result + maxSecondsBehindMaster = blm.maxSecondsBehindMaster_UNGUARDED() + binlogPlayersCount = int32(len(blm.players)) + return } diff --git a/go/vt/tabletmanager/healthcheck.go b/go/vt/tabletmanager/healthcheck.go index 10df3463ad..91290d4321 100644 --- a/go/vt/tabletmanager/healthcheck.go +++ b/go/vt/tabletmanager/healthcheck.go @@ -259,9 +259,7 @@ func (agent *ActionAgent) runHealthCheck(targetTabletType pbt.TabletType) { stats := &pb.RealtimeStats{ SecondsBehindMaster: uint32(replicationDelay.Seconds()), } - blpStats := agent.BinlogPlayerMap.StatusSummary() - stats.SecondsBehindMasterFilteredReplication = blpStats.maxSecondsBehindMaster - stats.BinlogPlayersCount = blpStats.binlogPlayersCount + stats.SecondsBehindMasterFilteredReplication, stats.BinlogPlayersCount = agent.BinlogPlayerMap.StatusSummary() if err != nil { stats.HealthError = err.Error() } From f8c8f9369228d74a70cda35c401d20d64d5ecfb0 Mon Sep 17 00:00:00 2001 From: Michael Berlin Date: Tue, 18 Aug 2015 13:24:24 -0700 Subject: [PATCH 42/80] tabletmanager: Fix golint warning that underscores are not allowed in Go names. --- go/vt/tabletmanager/binlog.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/go/vt/tabletmanager/binlog.go b/go/vt/tabletmanager/binlog.go index 98b24af866..1376f4ddfc 100644 --- a/go/vt/tabletmanager/binlog.go +++ b/go/vt/tabletmanager/binlog.go @@ -340,7 +340,7 @@ func RegisterBinlogPlayerMap(blm *BinlogPlayerMap) { stats.Publish("BinlogPlayerSecondsBehindMaster", stats.IntFunc(func() int64 { blm.mu.Lock() defer blm.mu.Unlock() - return blm.maxSecondsBehindMaster_UNGUARDED() + return blm.maxSecondsBehindMasterUNGUARDED() })) stats.Publish("BinlogPlayerSecondsBehindMasterMap", stats.CountersFunc(func() map[string]int64 { blm.mu.Lock() @@ -381,10 +381,10 @@ func (blm *BinlogPlayerMap) isRunningFilteredReplication() bool { return len(blm.players) != 0 } -// maxSecondsBehindMaster returns the maximum of the secondsBehindMaster +// maxSecondsBehindMasterUNGUARDED returns the maximum of the secondsBehindMaster // value of all binlog players i.e. the highest seen filtered replication lag. // NOTE: Caller must own a lock on blm.mu. -func (blm *BinlogPlayerMap) maxSecondsBehindMaster_UNGUARDED() int64 { +func (blm *BinlogPlayerMap) maxSecondsBehindMasterUNGUARDED() int64 { sbm := int64(0) for _, bpc := range blm.players { psbm := bpc.binlogPlayerStats.SecondsBehindMaster.Get() @@ -685,7 +685,7 @@ func (blm *BinlogPlayerMap) Status() *BinlogPlayerMapStatus { func (blm *BinlogPlayerMap) StatusSummary() (maxSecondsBehindMaster int64, binlogPlayersCount int32) { blm.mu.Lock() defer blm.mu.Unlock() - maxSecondsBehindMaster = blm.maxSecondsBehindMaster_UNGUARDED() + maxSecondsBehindMaster = blm.maxSecondsBehindMasterUNGUARDED() binlogPlayersCount = int32(len(blm.players)) return } From cd1821dc357d2b404331ec540386e1ead069f9de Mon Sep 17 00:00:00 2001 From: Michael Berlin Date: Tue, 18 Aug 2015 13:28:32 -0700 Subject: [PATCH 43/80] tabletserver: Add new RealTimeStats fields to the test. This ensures that conversions tests will fail because they test that all fields get properly converted. To fix the tests, we'll have to update the internal conversion. --- go/vt/tabletserver/tabletconntest/tabletconntest.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go/vt/tabletserver/tabletconntest/tabletconntest.go b/go/vt/tabletserver/tabletconntest/tabletconntest.go index aa4591ee62..169f87ac8b 100644 --- a/go/vt/tabletserver/tabletconntest/tabletconntest.go +++ b/go/vt/tabletserver/tabletconntest/tabletconntest.go @@ -990,6 +990,8 @@ var testStreamHealthStreamHealthResponse = &pb.StreamHealthResponse{ RealtimeStats: &pb.RealtimeStats{ HealthError: "random error", SecondsBehindMaster: 234, + BinlogPlayersCount: 1, + SecondsBehindMasterFilteredReplication: 2, CpuUsage: 1.0, }, } From 4809ba5bd48e96718459524c4227dbd89d70fe23 Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Tue, 18 Aug 2015 13:38:03 -0700 Subject: [PATCH 44/80] tabletserver upsert: Add clarification comment --- go/vt/tabletserver/query_executor.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go/vt/tabletserver/query_executor.go b/go/vt/tabletserver/query_executor.go index 93cecfe888..aabf745a1a 100644 --- a/go/vt/tabletserver/query_executor.go +++ b/go/vt/tabletserver/query_executor.go @@ -497,6 +497,8 @@ func (qre *QueryExecutor) execUpsertPK(conn poolConn, invalidator CacheInvalidat if !strings.Contains(terr.Message, "'PRIMARY'") { return nil, err } + // At this point, we know the insert failed due to a duplicate pk row. + // So, we just update the row. result, err = qre.execDMLPKRows(conn, qre.plan.UpsertQuery, pkRows, invalidator) if err != nil { return nil, err From e878386bcb36c4b512e034bc794983fcee024e33 Mon Sep 17 00:00:00 2001 From: Michael Berlin Date: Tue, 18 Aug 2015 14:39:03 -0700 Subject: [PATCH 45/80] Revert "Makefile: Enable debug logging for worker.py." This reverts commit 24348ab625ddf13f1335cef71bce8ed80e95247e. The intent of the commit was to find out why worker tests sometimes do not complete. We know the cause now: The worker command is hanging in State: finding target instances Here's an example log: https://s3.amazonaws.com/archive.travis-ci.org/jobs/75359822/log.txt --- Makefile | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index fd708de40f..e76afc9803 100644 --- a/Makefile +++ b/Makefile @@ -111,7 +111,6 @@ small_integration_test_files = \ initial_sharding_bytes.py \ initial_sharding.py -# TODO(mberlin): Remove -v option to worker.py when we found out what causes 10 minute Travis timeouts. medium_integration_test_files = \ tabletmanager.py \ reparent.py \ @@ -119,7 +118,7 @@ medium_integration_test_files = \ client_test.py \ vtgate_utils_test.py \ rowcache_invalidator.py \ - "worker.py -v" \ + worker.py \ automation_horizontal_resharding.py large_integration_test_files = \ @@ -146,19 +145,12 @@ SHELL = /bin/bash # function to execute a list of integration test files # exits on first failure -# TODO(mberlin): Remove special handling for worker.py when we found out what causes 10 minute Travis timeouts. define run_integration_tests cd test ; \ for t in $1 ; do \ echo $$(date): Running test/$$t... ; \ - if [[ $$t == *worker.py* ]]; then \ - time ./$$t $$VT_TEST_FLAGS 2>&1 ; \ - rc=$$? ; \ - else \ - output=$$(time ./$$t $$VT_TEST_FLAGS 2>&1) ; \ - rc=$$? ; \ - fi ; \ - if [[ $$rc != 0 ]]; then \ + output=$$(time ./$$t $$VT_TEST_FLAGS 2>&1) ; \ + if [[ $$? != 0 ]]; then \ echo "$$output" >&2 ; \ exit 1 ; \ fi ; \ From d4d9d3ee25743f483b515bb1649ab4096d6ee020 Mon Sep 17 00:00:00 2001 From: Michael Berlin Date: Tue, 18 Aug 2015 14:43:50 -0700 Subject: [PATCH 46/80] Makefile: Let each integration test run fail after a 5 minutes timeout. Currently, the worker tests are hanging from time to time. Travis kills these tests automatically after 10 minutes (of not receiving any output). We can be more aggressive about it and kill them ourselves after 5 minutes. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e76afc9803..65e64e80f0 100644 --- a/Makefile +++ b/Makefile @@ -149,7 +149,7 @@ define run_integration_tests cd test ; \ for t in $1 ; do \ echo $$(date): Running test/$$t... ; \ - output=$$(time ./$$t $$VT_TEST_FLAGS 2>&1) ; \ + output=$$(time timeout 5m ./$$t $$VT_TEST_FLAGS 2>&1) ; \ if [[ $$? != 0 ]]; then \ echo "$$output" >&2 ; \ exit 1 ; \ From 8c5a1db7a0d6e859e7374c09dc16f64ca62e15f9 Mon Sep 17 00:00:00 2001 From: Michael Berlin Date: Tue, 18 Aug 2015 16:16:57 -0700 Subject: [PATCH 47/80] tabletserver: Remove TODO. We decided that it's not necessary to implement it because it's no problem in practice. (The producer would have to close and drain the channel.) --- go/vt/tabletserver/grpcqueryservice/server.go | 1 - 1 file changed, 1 deletion(-) diff --git a/go/vt/tabletserver/grpcqueryservice/server.go b/go/vt/tabletserver/grpcqueryservice/server.go index 3c5debf84d..1f97fa7aa1 100644 --- a/go/vt/tabletserver/grpcqueryservice/server.go +++ b/go/vt/tabletserver/grpcqueryservice/server.go @@ -194,7 +194,6 @@ func (q *query) StreamHealth(request *pb.StreamHealthRequest, stream pbs.Query_S for shr := range c { // we send until the client disconnects if err := stream.Send(shr); err != nil { - // TODO(mberlin): Shouldn't we explicitly close c here? return } } From 6ca05d2a738bad67022c79f7ea3c8ac23c11656d Mon Sep 17 00:00:00 2001 From: Michael Berlin Date: Tue, 18 Aug 2015 17:18:37 -0700 Subject: [PATCH 48/80] automation: Add task WaitForFilteredReplication. --- .../automation/horizontal_resharding_task.go | 13 +++++++ go/vt/automation/scheduler.go | 2 ++ .../wait_for_filtered_replication_task.go | 35 +++++++++++++++++++ 3 files changed, 50 insertions(+) create mode 100644 go/vt/automation/wait_for_filtered_replication_task.go diff --git a/go/vt/automation/horizontal_resharding_task.go b/go/vt/automation/horizontal_resharding_task.go index f4a5753290..a059a14529 100644 --- a/go/vt/automation/horizontal_resharding_task.go +++ b/go/vt/automation/horizontal_resharding_task.go @@ -50,6 +50,19 @@ func (t *HorizontalReshardingTask) Run(parameters map[string]string) ([]*pb.Task } newTasks = append(newTasks, splitCloneTasks) + // TODO(mberlin): When the framework supports nesting tasks, these wait tasks should be run before each SplitDiff. + waitTasks := NewTaskContainer() + for _, destShard := range destShards { + AddTask(waitTasks, "WaitForFilteredReplicationTask", map[string]string{ + "keyspace": keyspace, + "shard": destShard, + "max_delay": "30s", + "vtctld_endpoint": vtctldEndpoint, + }) + } + newTasks = append(newTasks, waitTasks) + + // TODO(mberlin): Run all SplitDiffTasks in parallel which do not use overlapping source shards for the comparison. for _, destShard := range destShards { splitDiffTask := NewTaskContainer() AddTask(splitDiffTask, "SplitDiffTask", map[string]string{ diff --git a/go/vt/automation/scheduler.go b/go/vt/automation/scheduler.go index 7f4ca1eeb5..c4ab14d77d 100644 --- a/go/vt/automation/scheduler.go +++ b/go/vt/automation/scheduler.go @@ -207,6 +207,8 @@ func defaultTaskCreator(taskName string) Task { return &HorizontalReshardingTask{} case "CopySchemaShardTask": return &CopySchemaShardTask{} + case "WaitForFilteredReplicationTask": + return &WaitForFilteredReplicationTask{} case "SplitCloneTask": return &SplitCloneTask{} case "SplitDiffTask": diff --git a/go/vt/automation/wait_for_filtered_replication_task.go b/go/vt/automation/wait_for_filtered_replication_task.go new file mode 100644 index 0000000000..cc41876ad3 --- /dev/null +++ b/go/vt/automation/wait_for_filtered_replication_task.go @@ -0,0 +1,35 @@ +// Copyright 2015, Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package automation + +import ( + "fmt" + + pb "github.com/youtube/vitess/go/vt/proto/automation" + "golang.org/x/net/context" +) + +// WaitForFilteredReplicationTask runs vtctl WaitForFilteredReplication to block until the destination master +// (i.e. the receiving side of the filtered replication) has caught up up to max_delay with the source shard. +type WaitForFilteredReplicationTask struct { +} + +// Run is part of the Task interface. +func (t *WaitForFilteredReplicationTask) Run(parameters map[string]string) ([]*pb.TaskContainer, string, error) { + keyspaceAndShard := fmt.Sprintf("%v/%v", parameters["keyspace"], parameters["shard"]) + output, err := ExecuteVtctl(context.TODO(), parameters["vtctld_endpoint"], + []string{"WaitForFilteredReplication", "-max_delay", parameters["max_delay"], keyspaceAndShard}) + return nil, output, err +} + +// RequiredParameters is part of the Task interface. +func (t *WaitForFilteredReplicationTask) RequiredParameters() []string { + return []string{"keyspace", "shard", "max_delay", "vtctld_endpoint"} +} + +// OptionalParameters is part of the Task interface. +func (t *WaitForFilteredReplicationTask) OptionalParameters() []string { + return nil +} From 3a168820e2342716d738eafc9da4020238489605 Mon Sep 17 00:00:00 2001 From: Michael Berlin Date: Tue, 18 Aug 2015 17:19:54 -0700 Subject: [PATCH 49/80] automation: Client now prints the details of a successfully run operation. This output will be also visibile in automation_horizontal_resharding.py. --- go/cmd/automation_client/automation_client.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/go/cmd/automation_client/automation_client.go b/go/cmd/automation_client/automation_client.go index c8bcc27b66..c2da705000 100644 --- a/go/cmd/automation_client/automation_client.go +++ b/go/cmd/automation_client/automation_client.go @@ -85,16 +85,16 @@ func main() { os.Exit(4) } fmt.Println("Operation was enqueued. Details:", enqueueResponse) - errWait := waitForClusterOp(client, enqueueResponse.Id) + resp, errWait := waitForClusterOp(client, enqueueResponse.Id) if errWait != nil { fmt.Println("ERROR:", errWait) os.Exit(5) } - fmt.Println("SUCCESS: ClusterOperation finished.") + fmt.Printf("SUCCESS: ClusterOperation finished.\n\nDetails:\n%v", proto.MarshalTextString(resp)) } // waitForClusterOp polls and blocks until the ClusterOperation invocation specified by "id" has finished. If an error occured, it will be returned. -func waitForClusterOp(client pbs.AutomationClient, id string) error { +func waitForClusterOp(client pbs.AutomationClient, id string) (*pb.GetClusterOperationDetailsResponse, error) { for { req := &pb.GetClusterOperationDetailsRequest{ Id: id, @@ -102,17 +102,17 @@ func waitForClusterOp(client pbs.AutomationClient, id string) error { resp, err := client.GetClusterOperationDetails(context.Background(), req) if err != nil { - return fmt.Errorf("Failed to get ClusterOperation Details. Request: %v Error: %v", req, err) + return nil, fmt.Errorf("Failed to get ClusterOperation Details. Request: %v Error: %v", req, err) } switch resp.ClusterOp.State { case pb.ClusterOperationState_UNKNOWN_CLUSTER_OPERATION_STATE: - return fmt.Errorf("ClusterOperation is in an unknown state. Details: %v", resp) + return resp, fmt.Errorf("ClusterOperation is in an unknown state. Details: %v", resp) case pb.ClusterOperationState_CLUSTER_OPERATION_DONE: if resp.ClusterOp.Error != "" { - return fmt.Errorf("ClusterOperation failed. Details:\n%v", proto.MarshalTextString(resp)) + return resp, fmt.Errorf("ClusterOperation failed. Details:\n%v", proto.MarshalTextString(resp)) } - return nil + return resp, nil } time.Sleep(50 * time.Millisecond) From 2fb9e68dad1ec115991c838c9464a497802f0608 Mon Sep 17 00:00:00 2001 From: Michael Berlin Date: Tue, 18 Aug 2015 17:20:14 -0700 Subject: [PATCH 50/80] automation: Log each vtctlclient command. --- go/vt/automation/vtctlclient_wrapper.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/go/vt/automation/vtctlclient_wrapper.go b/go/vt/automation/vtctlclient_wrapper.go index 227c365b1e..9a57eb53ff 100644 --- a/go/vt/automation/vtctlclient_wrapper.go +++ b/go/vt/automation/vtctlclient_wrapper.go @@ -8,6 +8,8 @@ import ( "bytes" "time" + log "github.com/golang/glog" + "github.com/youtube/vitess/go/vt/logutil" "github.com/youtube/vitess/go/vt/vtctl/vtctlclient" "golang.org/x/net/context" @@ -17,6 +19,7 @@ import ( func ExecuteVtctl(ctx context.Context, server string, args []string) (string, error) { var output bytes.Buffer + log.Infof("Executing remote vtctl command: %v server: %v", args, server) err := vtctlclient.RunCommandAndWait( ctx, server, args, // TODO(mberlin): Should these values be configurable as flags? From db7daa47a12ec3abb8f0ceb56d673ba40817af90 Mon Sep 17 00:00:00 2001 From: Michael Berlin Date: Tue, 18 Aug 2015 17:26:19 -0700 Subject: [PATCH 51/80] vtctl: WaitForFilteredReplication no longer requires a "Target" message in the StreamHealthResponse. Unfortunately, "Target" is not set in SqlQuery for the destination master during resharding because its QueryService state is "not serving". Therefore, the health check does not include the "Target". I've removed the "Target" check for now to unblock the automation task. However, I think we should fix this and untangle the "Target" in SqlQuery from the serving state respectively the healthcheck. --- go/vt/tabletserver/sqlquery.go | 5 ---- go/vt/vtctl/vtctl.go | 10 ------- .../wait_for_filtered_replication_test.go | 27 +++---------------- 3 files changed, 4 insertions(+), 38 deletions(-) diff --git a/go/vt/tabletserver/sqlquery.go b/go/vt/tabletserver/sqlquery.go index adf501b73d..fa0b0c8b21 100644 --- a/go/vt/tabletserver/sqlquery.go +++ b/go/vt/tabletserver/sqlquery.go @@ -119,11 +119,6 @@ func NewSqlQuery(config Config) *SqlQuery { return sq } -// SetTargetForTest allows to modify the target without calling allowQueries. -func (sq *SqlQuery) SetTargetForTest(target *pb.Target) { - sq.target = target -} - // GetState returns the name of the current SqlQuery state. func (sq *SqlQuery) GetState() string { sq.mu.Lock() diff --git a/go/vt/vtctl/vtctl.go b/go/vt/vtctl/vtctl.go index 82e1ebb48b..594d694a5c 100644 --- a/go/vt/vtctl/vtctl.go +++ b/go/vt/vtctl/vtctl.go @@ -1388,16 +1388,6 @@ func commandWaitForFilteredReplication(ctx context.Context, wr *wrangler.Wrangle if !ok { return fmt.Errorf("stream ended early: %v", errFunc()) } - gotTarget := shr.Target - if gotTarget == nil { - return fmt.Errorf("stream health record did not include Target: %v", shr) - } - if gotTarget.Keyspace != keyspace || gotTarget.Shard != shard { - return fmt.Errorf("received health record for wrong tablet. Expected tablet: %v/%v received health record: %v", keyspace, shard, shr) - } - if gotTarget.TabletType != pb.TabletType_MASTER { - return fmt.Errorf("tablet: %v should be master, but is not. type: %v", alias, gotTarget.TabletType.String()) - } delaySecs := shr.RealtimeStats.SecondsBehindMasterFilteredReplication lastSeenDelay := time.Duration(delaySecs) * time.Second diff --git a/go/vt/wrangler/testlib/wait_for_filtered_replication_test.go b/go/vt/wrangler/testlib/wait_for_filtered_replication_test.go index abec8a81d9..8a45a3b78c 100644 --- a/go/vt/wrangler/testlib/wait_for_filtered_replication_test.go +++ b/go/vt/wrangler/testlib/wait_for_filtered_replication_test.go @@ -31,28 +31,10 @@ const destShard = "-80" // WaitForFilteredReplication ensures that the dest shard has caught up // with the source shard up to a maximum replication delay (in seconds). func TestWaitForFilteredReplication(t *testing.T) { - target := &pbq.Target{Keyspace: keyspace, Shard: destShard, TabletType: pbt.TabletType_MASTER} - - waitForFilteredReplicationDefaultDelay(t, target, "" /* expectedErr */) + waitForFilteredReplicationDefaultDelay(t, "" /* expectedErr */) } -// TestWaitForFilteredReplication_nonMasterFails tests that -// vtctl WaitForFilteredReplication fails if the queried tablet is not MASTER. -func TestWaitForFilteredReplication_nonMasterFails(t *testing.T) { - target := &pbq.Target{Keyspace: keyspace, Shard: destShard, TabletType: pbt.TabletType_REPLICA} - - waitForFilteredReplicationDefaultDelay(t, target, "should be master, but is not") -} - -// TestWaitForFilteredReplication_wrongTarget tests that -// vtctl WaitForFilteredReplication fails if the target is different than expected. -func TestWaitForFilteredReplication_wrongTarget(t *testing.T) { - target := &pbq.Target{Keyspace: keyspace, Shard: "wrongshard", TabletType: pbt.TabletType_MASTER} - - waitForFilteredReplicationDefaultDelay(t, target, "received health record for wrong tablet") -} - -func waitForFilteredReplicationDefaultDelay(t *testing.T, target *pbq.Target, expectedErr string) { +func waitForFilteredReplicationDefaultDelay(t *testing.T, expectedErr string) { // Replication is lagging behind. oneHourDelay := &pbq.RealtimeStats{ SecondsBehindMasterFilteredReplication: 3600, @@ -65,10 +47,10 @@ func waitForFilteredReplicationDefaultDelay(t *testing.T, target *pbq.Target, ex } } - waitForFilteredReplication(t, target, expectedErr, oneHourDelay, oneSecondDelayFunc) + waitForFilteredReplication(t, expectedErr, oneHourDelay, oneSecondDelayFunc) } -func waitForFilteredReplication(t *testing.T, target *pbq.Target, expectedErr string, initialStats *pbq.RealtimeStats, broadcastStatsFunc func() *pbq.RealtimeStats) { +func waitForFilteredReplication(t *testing.T, expectedErr string, initialStats *pbq.RealtimeStats, broadcastStatsFunc func() *pbq.RealtimeStats) { ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"}) wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), time.Second) vp := NewVtctlPipe(t, ts) @@ -92,7 +74,6 @@ func waitForFilteredReplication(t *testing.T, target *pbq.Target, expectedErr st testConfig.EnablePublishStats = false testConfig.DebugURLPrefix = fmt.Sprintf("TestWaitForFilteredReplication-%d-", rand.Int63()) qs := tabletserver.NewSqlQuery(testConfig) - qs.SetTargetForTest(target) grpcqueryservice.RegisterForTest(dest.RPCServer, qs) qs.BroadcastHealth(42, initialStats) From 316e5c1d1beabc351bfc04a5a518c12017207c6f Mon Sep 17 00:00:00 2001 From: Michael Berlin Date: Tue, 18 Aug 2015 17:30:46 -0700 Subject: [PATCH 52/80] vtctl: WaitForFilteredReplication fails when filtered replication is not running on the tablet. --- go/vt/vtctl/vtctl.go | 11 +++++++++- .../wait_for_filtered_replication_test.go | 22 ++++++++++++++----- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/go/vt/vtctl/vtctl.go b/go/vt/vtctl/vtctl.go index 594d694a5c..76926bd9a8 100644 --- a/go/vt/vtctl/vtctl.go +++ b/go/vt/vtctl/vtctl.go @@ -1389,7 +1389,16 @@ func commandWaitForFilteredReplication(ctx context.Context, wr *wrangler.Wrangle return fmt.Errorf("stream ended early: %v", errFunc()) } - delaySecs := shr.RealtimeStats.SecondsBehindMasterFilteredReplication + stats := shr.RealtimeStats + if stats == nil { + return fmt.Errorf("health record does not include RealtimeStats message. tablet: %v health record: %v", alias, shr) + } + + if stats.BinlogPlayersCount == 0 { + return fmt.Errorf("no filtered replication running on tablet: %v health record: %v", alias, shr) + } + + delaySecs := stats.SecondsBehindMasterFilteredReplication lastSeenDelay := time.Duration(delaySecs) * time.Second if lastSeenDelay < 0 { return fmt.Errorf("last seen delay should never be negative. tablet: %v delay: %v", alias, lastSeenDelay) diff --git a/go/vt/wrangler/testlib/wait_for_filtered_replication_test.go b/go/vt/wrangler/testlib/wait_for_filtered_replication_test.go index 8a45a3b78c..bc1688d9b3 100644 --- a/go/vt/wrangler/testlib/wait_for_filtered_replication_test.go +++ b/go/vt/wrangler/testlib/wait_for_filtered_replication_test.go @@ -31,23 +31,35 @@ const destShard = "-80" // WaitForFilteredReplication ensures that the dest shard has caught up // with the source shard up to a maximum replication delay (in seconds). func TestWaitForFilteredReplication(t *testing.T) { - waitForFilteredReplicationDefaultDelay(t, "" /* expectedErr */) -} - -func waitForFilteredReplicationDefaultDelay(t *testing.T, expectedErr string) { // Replication is lagging behind. oneHourDelay := &pbq.RealtimeStats{ + BinlogPlayersCount: 1, SecondsBehindMasterFilteredReplication: 3600, } // Replication caught up. oneSecondDelayFunc := func() *pbq.RealtimeStats { return &pbq.RealtimeStats{ + BinlogPlayersCount: 1, SecondsBehindMasterFilteredReplication: 1, } } - waitForFilteredReplication(t, expectedErr, oneHourDelay, oneSecondDelayFunc) + waitForFilteredReplication(t, "" /* expectedErr */, oneHourDelay, oneSecondDelayFunc) +} + +// TestWaitForFilteredReplication_noFilteredReplication checks that +// vtctl WaitForFilteredReplication fails when no filtered replication is +// running (judging by the tablet's returned stream health record). +func TestWaitForFilteredReplication_noFilteredReplication(t *testing.T) { + noFilteredReplication := &pbq.RealtimeStats{ + BinlogPlayersCount: 0, + } + noFilteredReplicationFunc := func() *pbq.RealtimeStats { + return noFilteredReplication + } + + waitForFilteredReplication(t, "no filtered replication running", noFilteredReplication, noFilteredReplicationFunc) } func waitForFilteredReplication(t *testing.T, expectedErr string, initialStats *pbq.RealtimeStats, broadcastStatsFunc func() *pbq.RealtimeStats) { From e3d3c19147a31fd397180da3942313850db0c33a Mon Sep 17 00:00:00 2001 From: Michael Berlin Date: Tue, 18 Aug 2015 17:34:04 -0700 Subject: [PATCH 53/80] vtctl: WaitForFilteredReplication fails when it saw multiple health records with an error in a row. We don't fail immediately because it may take some until healthcheck is run again and we observe a healthy state. --- go/vt/vtctl/vtctl.go | 19 ++++++++++++++++++- .../wait_for_filtered_replication_test.go | 13 +++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/go/vt/vtctl/vtctl.go b/go/vt/vtctl/vtctl.go index 76926bd9a8..7f8bdc7e32 100644 --- a/go/vt/vtctl/vtctl.go +++ b/go/vt/vtctl/vtctl.go @@ -1380,6 +1380,11 @@ func commandWaitForFilteredReplication(ctx context.Context, wr *wrangler.Wrangle return fmt.Errorf("could not stream health records from tablet: %v err: %v", alias, err) } var lastSeenDelay int + healthErrorInARow := 0 + // In case of automated reshardings, a tablet may still report itself as + // unhealthy e.g. because CopySchemaShard wasn't run yet and the db doesn't + // exist yet. + const allowedHealthErrorInARow = 3 for { select { case <-ctx.Done(): @@ -1388,11 +1393,23 @@ func commandWaitForFilteredReplication(ctx context.Context, wr *wrangler.Wrangle if !ok { return fmt.Errorf("stream ended early: %v", errFunc()) } - stats := shr.RealtimeStats if stats == nil { return fmt.Errorf("health record does not include RealtimeStats message. tablet: %v health record: %v", alias, shr) } + if stats.HealthError != "" { + healthErrorInARow++ + if healthErrorInARow >= allowedHealthErrorInARow { + return fmt.Errorf("tablet is not healthy. tablet: %v health record: %v", alias, shr) + } + wr.Logger().Printf("Tablet is not healthy. Waiting for %v more health"+ + " record(s) before the command will fail."+ + " tablet: %v health record: %v\n", + (allowedHealthErrorInARow - healthErrorInARow), alias, shr) + continue + } else { + healthErrorInARow = 0 + } if stats.BinlogPlayersCount == 0 { return fmt.Errorf("no filtered replication running on tablet: %v health record: %v", alias, shr) diff --git a/go/vt/wrangler/testlib/wait_for_filtered_replication_test.go b/go/vt/wrangler/testlib/wait_for_filtered_replication_test.go index bc1688d9b3..33e7350954 100644 --- a/go/vt/wrangler/testlib/wait_for_filtered_replication_test.go +++ b/go/vt/wrangler/testlib/wait_for_filtered_replication_test.go @@ -62,6 +62,19 @@ func TestWaitForFilteredReplication_noFilteredReplication(t *testing.T) { waitForFilteredReplication(t, "no filtered replication running", noFilteredReplication, noFilteredReplicationFunc) } +// TestWaitForFilteredReplication_unhealthy checks that +// vtctl WaitForFilteredReplication fails eventually when a tablet is not healthy. +func TestWaitForFilteredReplication_unhealthy(t *testing.T) { + unhealthy := &pbq.RealtimeStats{ + HealthError: "WaitForFilteredReplication: unhealthy test", + } + unhealthyFunc := func() *pbq.RealtimeStats { + return unhealthy + } + + waitForFilteredReplication(t, "tablet is not healthy", unhealthy, unhealthyFunc) +} + func waitForFilteredReplication(t *testing.T, expectedErr string, initialStats *pbq.RealtimeStats, broadcastStatsFunc func() *pbq.RealtimeStats) { ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"}) wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), time.Second) From c7ca8a329121e6646465917e4139de85a9ec6eae Mon Sep 17 00:00:00 2001 From: Anthony Yeh Date: Wed, 19 Aug 2015 00:19:37 -0700 Subject: [PATCH 54/80] web/vtctld: Fix typo. --- web/vtctld/shard.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/vtctld/shard.html b/web/vtctld/shard.html index a58ba687db..48aa255195 100644 --- a/web/vtctld/shard.html +++ b/web/vtctld/shard.html @@ -90,7 +90,7 @@
Seconds Behind Master - +
From 564436c966893d00afd6a69f325fa6c1a8ed218d Mon Sep 17 00:00:00 2001 From: Alain Jobart Date: Wed, 19 Aug 2015 08:02:32 -0700 Subject: [PATCH 55/80] Keyspace and Shard events in topo now only include proto3 structs. --- go/vt/etcdtopo/keyspace.go | 24 +++++++-------- go/vt/etcdtopo/shard.go | 30 +++++++++---------- go/vt/topo/events/keyspace_change.go | 5 ++-- go/vt/topo/events/keyspace_change_syslog.go | 4 +-- .../events/keyspace_change_syslog_test.go | 11 ++++--- go/vt/topo/events/shard_change.go | 8 +++-- go/vt/topo/events/shard_change_syslog.go | 4 +-- go/vt/topo/events/shard_change_syslog_test.go | 15 +++++++--- go/vt/zktopo/keyspace.go | 12 +++++--- go/vt/zktopo/shard.go | 18 +++++++---- 10 files changed, 74 insertions(+), 57 deletions(-) diff --git a/go/vt/etcdtopo/keyspace.go b/go/vt/etcdtopo/keyspace.go index 78dc57ee4d..db34e9cc81 100644 --- a/go/vt/etcdtopo/keyspace.go +++ b/go/vt/etcdtopo/keyspace.go @@ -26,23 +26,16 @@ func (s *Server) CreateKeyspace(ctx context.Context, keyspace string, value *pb. } global := s.getGlobal() - resp, err := global.Create(keyspaceFilePath(keyspace), string(data), 0 /* ttl */) - if err != nil { + if _, err = global.Create(keyspaceFilePath(keyspace), string(data), 0 /* ttl */); err != nil { return convertError(err) } - if err := initLockFile(global, keyspaceDirPath(keyspace)); err != nil { + if err = initLockFile(global, keyspaceDirPath(keyspace)); err != nil { return err } - // We don't return ErrBadResponse in this case because the Create() suceeeded - // and we don't really need the version to satisfy our contract - we're only - // logging it. - version := int64(-1) - if resp.Node != nil { - version = int64(resp.Node.ModifiedIndex) - } event.Dispatch(&events.KeyspaceChange{ - KeyspaceInfo: *topo.NewKeyspaceInfo(keyspace, value, version), + KeyspaceName: keyspace, + Keyspace: value, Status: "created", }) return nil @@ -65,7 +58,8 @@ func (s *Server) UpdateKeyspace(ctx context.Context, ki *topo.KeyspaceInfo, exis } event.Dispatch(&events.KeyspaceChange{ - KeyspaceInfo: *ki, + KeyspaceName: ki.KeyspaceName(), + Keyspace: ki.Keyspace, Status: "updated", }) return int64(resp.Node.ModifiedIndex), nil @@ -127,7 +121,8 @@ func (s *Server) DeleteKeyspaceShards(ctx context.Context, keyspace string) erro } event.Dispatch(&events.KeyspaceChange{ - KeyspaceInfo: *topo.NewKeyspaceInfo(keyspace, nil, -1), + KeyspaceName: keyspace, + Keyspace: nil, Status: "deleted all shards", }) return nil @@ -141,7 +136,8 @@ func (s *Server) DeleteKeyspace(ctx context.Context, keyspace string) error { } event.Dispatch(&events.KeyspaceChange{ - KeyspaceInfo: *topo.NewKeyspaceInfo(keyspace, nil, -1), + KeyspaceName: keyspace, + Keyspace: nil, Status: "deleted", }) return nil diff --git a/go/vt/etcdtopo/shard.go b/go/vt/etcdtopo/shard.go index 1d4eb99b7e..00033f5152 100644 --- a/go/vt/etcdtopo/shard.go +++ b/go/vt/etcdtopo/shard.go @@ -24,24 +24,18 @@ func (s *Server) CreateShard(ctx context.Context, keyspace, shard string, value } global := s.getGlobal() - resp, err := global.Create(shardFilePath(keyspace, shard), string(data), 0 /* ttl */) - if err != nil { + if _, err = global.Create(shardFilePath(keyspace, shard), string(data), 0 /* ttl */); err != nil { return convertError(err) } - if err := initLockFile(global, shardDirPath(keyspace, shard)); err != nil { + if err = initLockFile(global, shardDirPath(keyspace, shard)); err != nil { return err } - // We don't return ErrBadResponse in this case because the Create() suceeeded - // and we don't really need the version to satisfy our contract - we're only - // logging it. - version := int64(-1) - if resp.Node != nil { - version = int64(resp.Node.ModifiedIndex) - } event.Dispatch(&events.ShardChange{ - ShardInfo: *topo.NewShardInfo(keyspace, shard, value, version), - Status: "created", + KeyspaceName: keyspace, + ShardName: shard, + Shard: value, + Status: "created", }) return nil } @@ -63,8 +57,10 @@ func (s *Server) UpdateShard(ctx context.Context, si *topo.ShardInfo, existingVe } event.Dispatch(&events.ShardChange{ - ShardInfo: *si, - Status: "updated", + KeyspaceName: si.Keyspace(), + ShardName: si.ShardName(), + Shard: si.Shard, + Status: "updated", }) return int64(resp.Node.ModifiedIndex), nil } @@ -110,8 +106,10 @@ func (s *Server) DeleteShard(ctx context.Context, keyspace, shard string) error } event.Dispatch(&events.ShardChange{ - ShardInfo: *topo.NewShardInfo(keyspace, shard, nil, -1), - Status: "deleted", + KeyspaceName: keyspace, + ShardName: shard, + Shard: nil, + Status: "deleted", }) return nil } diff --git a/go/vt/topo/events/keyspace_change.go b/go/vt/topo/events/keyspace_change.go index d7a1e3e169..337e53c4ec 100644 --- a/go/vt/topo/events/keyspace_change.go +++ b/go/vt/topo/events/keyspace_change.go @@ -1,11 +1,12 @@ package events import ( - "github.com/youtube/vitess/go/vt/topo" + pb "github.com/youtube/vitess/go/vt/proto/topodata" ) // KeyspaceChange is an event that describes changes to a keyspace. type KeyspaceChange struct { - KeyspaceInfo topo.KeyspaceInfo + KeyspaceName string + Keyspace *pb.Keyspace Status string } diff --git a/go/vt/topo/events/keyspace_change_syslog.go b/go/vt/topo/events/keyspace_change_syslog.go index c7924e2ea4..b6bf15927f 100644 --- a/go/vt/topo/events/keyspace_change_syslog.go +++ b/go/vt/topo/events/keyspace_change_syslog.go @@ -9,8 +9,8 @@ import ( // Syslog writes the event to syslog. func (kc *KeyspaceChange) Syslog() (syslog.Priority, string) { - return syslog.LOG_INFO, fmt.Sprintf("%s [keyspace] %s", - kc.KeyspaceInfo.KeyspaceName(), kc.Status) + return syslog.LOG_INFO, fmt.Sprintf("%s [keyspace] %s value: %s", + kc.KeyspaceName, kc.Status, kc.Keyspace.String()) } var _ syslogger.Syslogger = (*KeyspaceChange)(nil) // compile-time interface check diff --git a/go/vt/topo/events/keyspace_change_syslog_test.go b/go/vt/topo/events/keyspace_change_syslog_test.go index ca766e2cfb..05b5a84407 100644 --- a/go/vt/topo/events/keyspace_change_syslog_test.go +++ b/go/vt/topo/events/keyspace_change_syslog_test.go @@ -4,14 +4,17 @@ import ( "log/syslog" "testing" - "github.com/youtube/vitess/go/vt/topo" + pb "github.com/youtube/vitess/go/vt/proto/topodata" ) func TestKeyspaceChangeSyslog(t *testing.T) { - wantSev, wantMsg := syslog.LOG_INFO, "keyspace-123 [keyspace] status" + wantSev, wantMsg := syslog.LOG_INFO, "keyspace-123 [keyspace] status value: sharding_column_name:\"sharded_by_me\" " kc := &KeyspaceChange{ - KeyspaceInfo: *topo.NewKeyspaceInfo("keyspace-123", nil, -1), - Status: "status", + KeyspaceName: "keyspace-123", + Keyspace: &pb.Keyspace{ + ShardingColumnName: "sharded_by_me", + }, + Status: "status", } gotSev, gotMsg := kc.Syslog() diff --git a/go/vt/topo/events/shard_change.go b/go/vt/topo/events/shard_change.go index b68603ab40..596aaa4e7f 100644 --- a/go/vt/topo/events/shard_change.go +++ b/go/vt/topo/events/shard_change.go @@ -1,11 +1,13 @@ package events import ( - "github.com/youtube/vitess/go/vt/topo" + pb "github.com/youtube/vitess/go/vt/proto/topodata" ) // ShardChange is an event that describes changes to a shard. type ShardChange struct { - ShardInfo topo.ShardInfo - Status string + KeyspaceName string + ShardName string + Shard *pb.Shard + Status string } diff --git a/go/vt/topo/events/shard_change_syslog.go b/go/vt/topo/events/shard_change_syslog.go index b8e10df757..f033ff7da7 100644 --- a/go/vt/topo/events/shard_change_syslog.go +++ b/go/vt/topo/events/shard_change_syslog.go @@ -9,8 +9,8 @@ import ( // Syslog writes the event to syslog. func (sc *ShardChange) Syslog() (syslog.Priority, string) { - return syslog.LOG_INFO, fmt.Sprintf("%s/%s [shard] %s", - sc.ShardInfo.Keyspace(), sc.ShardInfo.ShardName(), sc.Status) + return syslog.LOG_INFO, fmt.Sprintf("%s/%s [shard] %s value: %s", + sc.KeyspaceName, sc.ShardName, sc.Status, sc.Shard.String()) } var _ syslogger.Syslogger = (*ShardChange)(nil) // compile-time interface check diff --git a/go/vt/topo/events/shard_change_syslog_test.go b/go/vt/topo/events/shard_change_syslog_test.go index 48ed942feb..b033e22c54 100644 --- a/go/vt/topo/events/shard_change_syslog_test.go +++ b/go/vt/topo/events/shard_change_syslog_test.go @@ -4,14 +4,21 @@ import ( "log/syslog" "testing" - "github.com/youtube/vitess/go/vt/topo" + pb "github.com/youtube/vitess/go/vt/proto/topodata" ) func TestShardChangeSyslog(t *testing.T) { - wantSev, wantMsg := syslog.LOG_INFO, "keyspace-123/shard-123 [shard] status" + wantSev, wantMsg := syslog.LOG_INFO, "keyspace-123/shard-123 [shard] status value: master_alias: " sc := &ShardChange{ - ShardInfo: *topo.NewShardInfo("keyspace-123", "shard-123", nil, -1), - Status: "status", + KeyspaceName: "keyspace-123", + ShardName: "shard-123", + Shard: &pb.Shard{ + MasterAlias: &pb.TabletAlias{ + Cell: "test", + Uid: 123, + }, + }, + Status: "status", } gotSev, gotMsg := sc.Syslog() diff --git a/go/vt/zktopo/keyspace.go b/go/vt/zktopo/keyspace.go index 75ce431110..1c3908df19 100644 --- a/go/vt/zktopo/keyspace.go +++ b/go/vt/zktopo/keyspace.go @@ -63,7 +63,8 @@ func (zkts *Server) CreateKeyspace(ctx context.Context, keyspace string, value * } event.Dispatch(&events.KeyspaceChange{ - KeyspaceInfo: *topo.NewKeyspaceInfo(keyspace, value, -1), + KeyspaceName: keyspace, + Keyspace: value, Status: "created", }) return nil @@ -85,7 +86,8 @@ func (zkts *Server) UpdateKeyspace(ctx context.Context, ki *topo.KeyspaceInfo, e } event.Dispatch(&events.KeyspaceChange{ - KeyspaceInfo: *ki, + KeyspaceName: ki.KeyspaceName(), + Keyspace: ki.Keyspace, Status: "updated", }) return int64(stat.Version()), nil @@ -103,7 +105,8 @@ func (zkts *Server) DeleteKeyspace(ctx context.Context, keyspace string) error { } event.Dispatch(&events.KeyspaceChange{ - KeyspaceInfo: *topo.NewKeyspaceInfo(keyspace, nil, -1), + KeyspaceName: keyspace, + Keyspace: nil, Status: "deleted", }) return nil @@ -150,7 +153,8 @@ func (zkts *Server) DeleteKeyspaceShards(ctx context.Context, keyspace string) e } event.Dispatch(&events.KeyspaceChange{ - KeyspaceInfo: *topo.NewKeyspaceInfo(keyspace, nil, -1), + KeyspaceName: keyspace, + Keyspace: nil, Status: "deleted all shards", }) return nil diff --git a/go/vt/zktopo/shard.go b/go/vt/zktopo/shard.go index 71d7832520..3337466e37 100644 --- a/go/vt/zktopo/shard.go +++ b/go/vt/zktopo/shard.go @@ -58,8 +58,10 @@ func (zkts *Server) CreateShard(ctx context.Context, keyspace, shard string, val } event.Dispatch(&events.ShardChange{ - ShardInfo: *topo.NewShardInfo(keyspace, shard, value, -1), - Status: "created", + KeyspaceName: keyspace, + ShardName: shard, + Shard: value, + Status: "created", }) return nil } @@ -80,8 +82,10 @@ func (zkts *Server) UpdateShard(ctx context.Context, si *topo.ShardInfo, existin } event.Dispatch(&events.ShardChange{ - ShardInfo: *si, - Status: "updated", + KeyspaceName: si.Keyspace(), + ShardName: si.ShardName(), + Shard: si.Shard, + Status: "updated", }) return int64(stat.Version()), nil } @@ -148,8 +152,10 @@ func (zkts *Server) DeleteShard(ctx context.Context, keyspace, shard string) err } event.Dispatch(&events.ShardChange{ - ShardInfo: *topo.NewShardInfo(keyspace, shard, nil, -1), - Status: "deleted", + KeyspaceName: keyspace, + ShardName: shard, + Shard: nil, + Status: "deleted", }) return nil } From 25f9cbe4598d4f2dac354c77d34a531bce0e17e2 Mon Sep 17 00:00:00 2001 From: Alain Jobart Date: Wed, 19 Aug 2015 10:23:22 -0700 Subject: [PATCH 56/80] Moving proto-related Tablet methods to go/vt/topo/topoproto. That way we don't need to depend on the whole go/vt/topo module to print a tablet alias, for instance. --- go/cmd/vtctld/actions.go | 3 +- go/cmd/vtctld/api.go | 7 +- go/cmd/vtctld/explorer.go | 6 +- go/cmd/vtctld/plugin_zktopo.go | 3 +- go/cmd/vtctld/vtctld.go | 3 +- go/cmd/vttablet/vttablet.go | 3 +- go/vt/client/client.go | 4 +- go/vt/etcdtopo/config.go | 4 +- go/vt/etcdtopo/serving_graph.go | 3 +- go/vt/etcdtopo/tablet.go | 3 +- go/vt/tabletmanager/after_action.go | 5 +- go/vt/tabletmanager/agent.go | 3 +- go/vt/tabletmanager/agent_rpc_actions.go | 5 +- go/vt/tabletmanager/binlog.go | 7 +- go/vt/tabletmanager/healthcheck.go | 3 +- go/vt/tabletmanager/init_tablet.go | 5 +- go/vt/tabletmanager/reparent.go | 5 +- go/vt/tabletmanager/rpc_server.go | 10 +- go/vt/topo/events/tablet_change_syslog.go | 4 +- go/vt/topo/proto3.go | 3 +- go/vt/topo/replication.go | 3 +- go/vt/topo/shard.go | 42 ++-- go/vt/topo/tablet.go | 186 +----------------- go/vt/topo/topoproto/tablet.go | 180 +++++++++++++++++ go/vt/topotools/events/reparent_syslog.go | 6 +- go/vt/topotools/tablet.go | 3 +- go/vt/topotools/topology.go | 3 +- go/vt/topotools/topology_test.go | 7 +- go/vt/topotools/utils.go | 3 +- go/vt/vtctl/query.go | 19 +- go/vt/vtctl/reparent.go | 11 +- go/vt/vtctl/vtctl.go | 75 +++---- go/vt/worker/clone_utils.go | 7 +- go/vt/worker/diff_utils.go | 5 +- go/vt/worker/split_clone.go | 15 +- go/vt/worker/split_diff.go | 7 +- go/vt/worker/sqldiffer.go | 14 +- go/vt/worker/topo_utils.go | 5 +- go/vt/worker/vertical_split_clone.go | 17 +- go/vt/worker/vertical_split_diff.go | 37 ++-- go/vt/wrangler/cleaner.go | 11 +- go/vt/wrangler/keyspace.go | 13 +- go/vt/wrangler/permissions.go | 19 +- go/vt/wrangler/reparent.go | 101 +++++----- go/vt/wrangler/schema.go | 33 ++-- go/vt/wrangler/shard.go | 21 +- go/vt/wrangler/tablet.go | 19 +- go/vt/wrangler/testlib/backup_test.go | 4 +- .../testlib/copy_schema_shard_test.go | 4 +- .../testlib/emergency_reparent_shard_test.go | 4 +- .../testlib/init_shard_master_test.go | 7 +- .../testlib/planned_reparent_shard_test.go | 4 +- .../testlib/reparent_external_test.go | 9 +- go/vt/wrangler/testlib/reparent_utils_test.go | 7 +- go/vt/wrangler/validator.go | 25 +-- go/vt/wrangler/version.go | 19 +- go/vt/wrangler/zkns.go | 3 +- go/vt/zktopo/serving_graph.go | 3 +- go/vt/zktopo/tablet.go | 5 +- 59 files changed, 553 insertions(+), 492 deletions(-) create mode 100644 go/vt/topo/topoproto/tablet.go diff --git a/go/cmd/vtctld/actions.go b/go/cmd/vtctld/actions.go index 81280646cb..6bd857f81e 100644 --- a/go/cmd/vtctld/actions.go +++ b/go/cmd/vtctld/actions.go @@ -14,6 +14,7 @@ import ( "github.com/youtube/vitess/go/vt/tabletmanager/actionnode" "github.com/youtube/vitess/go/vt/tabletmanager/tmclient" "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "github.com/youtube/vitess/go/vt/wrangler" pb "github.com/youtube/vitess/go/vt/proto/topodata" @@ -143,7 +144,7 @@ func (ar *ActionRepository) ApplyShardAction(ctx context.Context, actionName, ke func (ar *ActionRepository) ApplyTabletAction(ctx context.Context, actionName string, tabletAlias *pb.TabletAlias, r *http.Request) *ActionResult { result := &ActionResult{ Name: actionName, - Parameters: topo.TabletAliasString(tabletAlias), + Parameters: topoproto.TabletAliasString(tabletAlias), } action, ok := ar.tabletActions[actionName] diff --git a/go/cmd/vtctld/api.go b/go/cmd/vtctld/api.go index 92707e6c56..6def409ced 100644 --- a/go/cmd/vtctld/api.go +++ b/go/cmd/vtctld/api.go @@ -12,6 +12,7 @@ import ( "github.com/youtube/vitess/go/vt/schemamanager" "github.com/youtube/vitess/go/vt/tabletmanager/tmclient" "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "golang.org/x/net/context" ) @@ -177,14 +178,14 @@ func initAPI(ctx context.Context, ts topo.Server, actions *ActionRepository) { // Get tablet health. if parts := strings.Split(tabletPath, "/"); len(parts) == 2 && parts[1] == "health" { - tabletAlias, err := topo.ParseTabletAliasString(parts[0]) + tabletAlias, err := topoproto.ParseTabletAlias(parts[0]) if err != nil { return nil, err } return tabletHealthCache.Get(ctx, tabletAlias) } - tabletAlias, err := topo.ParseTabletAliasString(tabletPath) + tabletAlias, err := topoproto.ParseTabletAlias(tabletPath) if err != nil { return nil, err } @@ -219,7 +220,7 @@ func initAPI(ctx context.Context, ts topo.Server, actions *ActionRepository) { return ts.GetSrvTabletTypesPerShard(ctx, parts[0], parts[1], parts[2]) } - tabletType, err := topo.ParseTabletType(parts[3]) + tabletType, err := topoproto.ParseTabletType(parts[3]) if err != nil { return nil, fmt.Errorf("invalid tablet type %v: %v", parts[3], err) } diff --git a/go/cmd/vtctld/explorer.go b/go/cmd/vtctld/explorer.go index 5444d5f207..64aba89561 100644 --- a/go/cmd/vtctld/explorer.go +++ b/go/cmd/vtctld/explorer.go @@ -10,7 +10,7 @@ import ( log "github.com/golang/glog" "github.com/youtube/vitess/go/cmd/vtctld/proto" - "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" pb "github.com/youtube/vitess/go/vt/proto/topodata" ) @@ -134,7 +134,7 @@ func handleExplorerRedirect(r *http.Request) (string, error) { if keyspace == "" || shard == "" || cell == "" || tabletType == "" { return "", errors.New("keyspace, shard, cell, and tablet_type are required for this redirect") } - tt, err := topo.ParseTabletType(tabletType) + tt, err := topoproto.ParseTabletType(tabletType) if err != nil { return "", fmt.Errorf("cannot parse tablet type %v: %v", tabletType, err) } @@ -144,7 +144,7 @@ func handleExplorerRedirect(r *http.Request) (string, error) { if alias == "" { return "", errors.New("alias is required for this redirect") } - tabletAlias, err := topo.ParseTabletAliasString(alias) + tabletAlias, err := topoproto.ParseTabletAlias(alias) if err != nil { return "", fmt.Errorf("bad tablet alias %q: %v", alias, err) } diff --git a/go/cmd/vtctld/plugin_zktopo.go b/go/cmd/vtctld/plugin_zktopo.go index 6bc0bb4d20..1b7fa3d223 100644 --- a/go/cmd/vtctld/plugin_zktopo.go +++ b/go/cmd/vtctld/plugin_zktopo.go @@ -19,6 +19,7 @@ import ( "github.com/youtube/vitess/go/netutil" "github.com/youtube/vitess/go/vt/servenv" "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "github.com/youtube/vitess/go/vt/zktopo" "github.com/youtube/vitess/go/zk" @@ -71,7 +72,7 @@ func (ex ZkExplorer) GetSrvTypePath(cell, keyspace, shard string, tabletType pb. // GetTabletPath is part of the Explorer interface func (ex ZkExplorer) GetTabletPath(alias *pb.TabletAlias) string { - return path.Join("/zk", alias.Cell, "vt/tablets", topo.TabletAliasUIDStr(alias)) + return path.Join("/zk", alias.Cell, "vt/tablets", topoproto.TabletAliasUIDStr(alias)) } // GetReplicationSlaves is part of the Explorer interface diff --git a/go/cmd/vtctld/vtctld.go b/go/cmd/vtctld/vtctld.go index cb766512ff..d771479059 100644 --- a/go/cmd/vtctld/vtctld.go +++ b/go/cmd/vtctld/vtctld.go @@ -17,6 +17,7 @@ import ( "github.com/youtube/vitess/go/vt/servenv" "github.com/youtube/vitess/go/vt/tabletmanager/tmclient" "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "github.com/youtube/vitess/go/vt/topotools" "github.com/youtube/vitess/go/vt/wrangler" @@ -241,7 +242,7 @@ func main() { http.Error(w, "no alias provided", http.StatusBadRequest) return } - tabletAlias, err := topo.ParseTabletAliasString(alias) + tabletAlias, err := topoproto.ParseTabletAlias(alias) if err != nil { http.Error(w, "bad alias provided", http.StatusBadRequest) return diff --git a/go/cmd/vttablet/vttablet.go b/go/cmd/vttablet/vttablet.go index 2ed96b07c7..bfe4994082 100644 --- a/go/cmd/vttablet/vttablet.go +++ b/go/cmd/vttablet/vttablet.go @@ -20,6 +20,7 @@ import ( "github.com/youtube/vitess/go/vt/tabletmanager/actionnode" "github.com/youtube/vitess/go/vt/tabletserver" "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "golang.org/x/net/context" // import mysql to register mysql connection function @@ -66,7 +67,7 @@ func main() { log.Errorf("tabletPath required") exit.Return(1) } - tabletAlias, err := topo.ParseTabletAliasString(*tabletPath) + tabletAlias, err := topoproto.ParseTabletAlias(*tabletPath) if err != nil { log.Error(err) diff --git a/go/vt/client/client.go b/go/vt/client/client.go index a6d7c053cd..f58d64db80 100644 --- a/go/vt/client/client.go +++ b/go/vt/client/client.go @@ -13,7 +13,7 @@ import ( "time" mproto "github.com/youtube/vitess/go/mysql/proto" - "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "github.com/youtube/vitess/go/vt/vtgate/vtgateconn" "golang.org/x/net/context" @@ -41,7 +41,7 @@ func (d drv) Open(name string) (driver.Conn, error) { if err != nil { return nil, err } - c.tabletType, err = topo.ParseTabletType(c.TabletType) + c.tabletType, err = topoproto.ParseTabletType(c.TabletType) if err != nil { return nil, err } diff --git a/go/vt/etcdtopo/config.go b/go/vt/etcdtopo/config.go index c6b48a399c..8d05cd5e78 100644 --- a/go/vt/etcdtopo/config.go +++ b/go/vt/etcdtopo/config.go @@ -12,7 +12,7 @@ import ( "github.com/youtube/vitess/go/flagutil" pb "github.com/youtube/vitess/go/vt/proto/topodata" - "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" ) const ( @@ -70,7 +70,7 @@ func shardFilePath(keyspace, shard string) string { } func tabletDirPath(tabletAlias *pb.TabletAlias) string { - return path.Join(tabletsDirPath, topo.TabletAliasString(tabletAlias)) + return path.Join(tabletsDirPath, topoproto.TabletAliasString(tabletAlias)) } func tabletFilePath(tabletAlias *pb.TabletAlias) string { diff --git a/go/vt/etcdtopo/serving_graph.go b/go/vt/etcdtopo/serving_graph.go index a158dd8d5a..bd5caf7edd 100644 --- a/go/vt/etcdtopo/serving_graph.go +++ b/go/vt/etcdtopo/serving_graph.go @@ -13,6 +13,7 @@ import ( "github.com/coreos/go-etcd/etcd" log "github.com/golang/glog" "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "golang.org/x/net/context" pb "github.com/youtube/vitess/go/vt/proto/topodata" @@ -41,7 +42,7 @@ func (s *Server) GetSrvTabletTypesPerShard(ctx context.Context, cellName, keyspa tabletTypes := make([]pb.TabletType, 0, len(resp.Node.Nodes)) for _, n := range resp.Node.Nodes { strType := path.Base(n.Key) - if tt, err := topo.ParseTabletType(strType); err == nil { + if tt, err := topoproto.ParseTabletType(strType); err == nil { tabletTypes = append(tabletTypes, tt) } } diff --git a/go/vt/etcdtopo/tablet.go b/go/vt/etcdtopo/tablet.go index 4b288e7a39..ead6967730 100644 --- a/go/vt/etcdtopo/tablet.go +++ b/go/vt/etcdtopo/tablet.go @@ -11,6 +11,7 @@ import ( "github.com/youtube/vitess/go/event" "github.com/youtube/vitess/go/vt/topo" "github.com/youtube/vitess/go/vt/topo/events" + "github.com/youtube/vitess/go/vt/topo/topoproto" "golang.org/x/net/context" pb "github.com/youtube/vitess/go/vt/proto/topodata" @@ -171,7 +172,7 @@ func (s *Server) GetTabletsByCell(ctx context.Context, cellName string) ([]*pb.T tablets := make([]*pb.TabletAlias, 0, len(nodes)) for _, node := range nodes { - tabletAlias, err := topo.ParseTabletAliasString(node) + tabletAlias, err := topoproto.ParseTabletAlias(node) if err != nil { return nil, err } diff --git a/go/vt/tabletmanager/after_action.go b/go/vt/tabletmanager/after_action.go index b857966f63..236501de7c 100644 --- a/go/vt/tabletmanager/after_action.go +++ b/go/vt/tabletmanager/after_action.go @@ -23,6 +23,7 @@ import ( "github.com/youtube/vitess/go/vt/tabletserver" "github.com/youtube/vitess/go/vt/tabletserver/planbuilder" "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" pb "github.com/youtube/vitess/go/vt/proto/query" pbt "github.com/youtube/vitess/go/vt/proto/topodata" @@ -56,7 +57,7 @@ func (agent *ActionAgent) allowQueries(tablet *pbt.Tablet, blacklistedTables []s if agent.DBConfigs != nil { // Update our DB config to match the info we have in the tablet if agent.DBConfigs.App.DbName == "" { - agent.DBConfigs.App.DbName = topo.TabletDbName(tablet) + agent.DBConfigs.App.DbName = topoproto.TabletDbName(tablet) } agent.DBConfigs.App.Keyspace = tablet.Keyspace agent.DBConfigs.App.Shard = tablet.Shard @@ -116,7 +117,7 @@ func (agent *ActionAgent) loadKeyspaceAndBlacklistRules(tablet *pbt.Tablet, blac blacklistRules := tabletserver.NewQueryRules() if len(blacklistedTables) > 0 { // tables, first resolve wildcards - tables, err := mysqlctl.ResolveTables(agent.MysqlDaemon, topo.TabletDbName(tablet), blacklistedTables) + tables, err := mysqlctl.ResolveTables(agent.MysqlDaemon, topoproto.TabletDbName(tablet), blacklistedTables) if err != nil { return err } diff --git a/go/vt/tabletmanager/agent.go b/go/vt/tabletmanager/agent.go index f549917928..26ec7f13e1 100644 --- a/go/vt/tabletmanager/agent.go +++ b/go/vt/tabletmanager/agent.go @@ -44,6 +44,7 @@ import ( "github.com/youtube/vitess/go/vt/tabletmanager/events" "github.com/youtube/vitess/go/vt/tabletserver" "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "github.com/youtube/vitess/go/vt/topotools" pb "github.com/youtube/vitess/go/vt/proto/topodata" @@ -472,7 +473,7 @@ func (agent *ActionAgent) Stop() { // hookExtraEnv returns the map to pass to local hooks func (agent *ActionAgent) hookExtraEnv() map[string]string { - return map[string]string{"TABLET_ALIAS": topo.TabletAliasString(agent.TabletAlias)} + return map[string]string{"TABLET_ALIAS": topoproto.TabletAliasString(agent.TabletAlias)} } // checkTabletMysqlPort will check the mysql port for the tablet is good, diff --git a/go/vt/tabletmanager/agent_rpc_actions.go b/go/vt/tabletmanager/agent_rpc_actions.go index fa379152bc..5cdc7ecacf 100644 --- a/go/vt/tabletmanager/agent_rpc_actions.go +++ b/go/vt/tabletmanager/agent_rpc_actions.go @@ -16,6 +16,7 @@ import ( myproto "github.com/youtube/vitess/go/vt/mysqlctl/proto" "github.com/youtube/vitess/go/vt/tabletmanager/actionnode" "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "github.com/youtube/vitess/go/vt/topotools" "golang.org/x/net/context" @@ -426,7 +427,7 @@ func (agent *ActionAgent) InitMaster(ctx context.Context) (myproto.ReplicationPo // PopulateReparentJournal adds an entry into the reparent_journal table. func (agent *ActionAgent) PopulateReparentJournal(ctx context.Context, timeCreatedNS int64, actionName string, masterAlias *pb.TabletAlias, pos myproto.ReplicationPosition) error { cmds := mysqlctl.CreateReparentJournal() - cmds = append(cmds, mysqlctl.PopulateReparentJournal(timeCreatedNS, actionName, topo.TabletAliasString(masterAlias), pos)) + cmds = append(cmds, mysqlctl.PopulateReparentJournal(timeCreatedNS, actionName, topoproto.TabletAliasString(masterAlias), pos)) return agent.MysqlDaemon.ExecuteSuperQueryList(cmds) } @@ -708,7 +709,7 @@ func (agent *ActionAgent) Backup(ctx context.Context, concurrency int, logger lo // now we can run the backup bucket := fmt.Sprintf("%v/%v", tablet.Keyspace, tablet.Shard) - name := fmt.Sprintf("%v.%v", topo.TabletAliasString(tablet.Alias), time.Now().UTC().Format("2006-01-02.150405")) + name := fmt.Sprintf("%v.%v", topoproto.TabletAliasString(tablet.Alias), time.Now().UTC().Format("2006-01-02.150405")) returnErr := mysqlctl.Backup(ctx, agent.MysqlDaemon, l, bucket, name, concurrency, agent.hookExtraEnv()) // and change our type back to the appropriate value: diff --git a/go/vt/tabletmanager/binlog.go b/go/vt/tabletmanager/binlog.go index 1376f4ddfc..d93e08e08b 100644 --- a/go/vt/tabletmanager/binlog.go +++ b/go/vt/tabletmanager/binlog.go @@ -27,6 +27,7 @@ import ( "github.com/youtube/vitess/go/vt/mysqlctl" myproto "github.com/youtube/vitess/go/vt/mysqlctl/proto" "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "golang.org/x/net/context" pb "github.com/youtube/vitess/go/vt/proto/topodata" @@ -445,7 +446,7 @@ func (blm *BinlogPlayerMap) RefreshMap(ctx context.Context, tablet *pb.Tablet, k blm.mu.Lock() if blm.dbConfig.DbName == "" { - blm.dbConfig.DbName = topo.TabletDbName(tablet) + blm.dbConfig.DbName = topoproto.TabletDbName(tablet) } // get the existing sources and build a map of sources to remove @@ -458,7 +459,7 @@ func (blm *BinlogPlayerMap) RefreshMap(ctx context.Context, tablet *pb.Tablet, k // for each source, add it if not there, and delete from toRemove for _, sourceShard := range shardInfo.SourceShards { - blm.addPlayer(ctx, tablet.Alias.Cell, keyspaceInfo.ShardingColumnType, tablet.KeyRange, sourceShard, topo.TabletDbName(tablet)) + blm.addPlayer(ctx, tablet.Alias.Cell, keyspaceInfo.ShardingColumnType, tablet.KeyRange, sourceShard, topoproto.TabletDbName(tablet)) delete(toRemove, sourceShard.Uid) } hasPlayers := len(shardInfo.SourceShards) > 0 @@ -604,7 +605,7 @@ func (bpcs *BinlogPlayerControllerStatus) SourceShardAsHTML() template.HTML { // SourceTabletAlias returns the string version of the SourceTablet alias, if set func (bpcs *BinlogPlayerControllerStatus) SourceTabletAlias() string { if bpcs.SourceTablet != nil { - return topo.TabletAliasString(bpcs.SourceTablet) + return topoproto.TabletAliasString(bpcs.SourceTablet) } return "" } diff --git a/go/vt/tabletmanager/healthcheck.go b/go/vt/tabletmanager/healthcheck.go index 91290d4321..f8de8a717f 100644 --- a/go/vt/tabletmanager/healthcheck.go +++ b/go/vt/tabletmanager/healthcheck.go @@ -19,6 +19,7 @@ import ( "github.com/youtube/vitess/go/timer" "github.com/youtube/vitess/go/vt/servenv" "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "github.com/youtube/vitess/go/vt/topotools" pb "github.com/youtube/vitess/go/vt/proto/query" @@ -103,7 +104,7 @@ func (agent *ActionAgent) initHealthCheck() { return } - tt, err := topo.ParseTabletType(*targetTabletType) + tt, err := topoproto.ParseTabletType(*targetTabletType) if err != nil { log.Fatalf("Invalid target tablet type %v: %v", *targetTabletType, err) } diff --git a/go/vt/tabletmanager/init_tablet.go b/go/vt/tabletmanager/init_tablet.go index 80a597e8c0..24ee26f3f5 100644 --- a/go/vt/tabletmanager/init_tablet.go +++ b/go/vt/tabletmanager/init_tablet.go @@ -18,6 +18,7 @@ import ( "github.com/youtube/vitess/go/netutil" "github.com/youtube/vitess/go/vt/tabletmanager/actionnode" "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "github.com/youtube/vitess/go/vt/topotools" "golang.org/x/net/context" @@ -55,7 +56,7 @@ func (agent *ActionAgent) InitTablet(port, gRPCPort int32) error { // use the type specified on the command line var err error - tabletType, err = topo.ParseTabletType(*initTabletType) + tabletType, err = topoproto.ParseTabletType(*initTabletType) if err != nil { log.Fatalf("Invalid init tablet type %v: %v", *initTabletType, err) } @@ -102,7 +103,7 @@ func (agent *ActionAgent) InitTablet(port, gRPCPort int32) error { if err != nil { return fmt.Errorf("InitTablet cannot GetOrCreateShard shard: %v", err) } - if si.MasterAlias != nil && topo.TabletAliasEqual(si.MasterAlias, agent.TabletAlias) { + if si.MasterAlias != nil && topoproto.TabletAliasEqual(si.MasterAlias, agent.TabletAlias) { // we are the current master for this shard (probably // means the master tablet process was just restarted), // so InitTablet as master. diff --git a/go/vt/tabletmanager/reparent.go b/go/vt/tabletmanager/reparent.go index 9298d9db23..110e684f46 100644 --- a/go/vt/tabletmanager/reparent.go +++ b/go/vt/tabletmanager/reparent.go @@ -17,6 +17,7 @@ import ( "github.com/youtube/vitess/go/vt/concurrency" "github.com/youtube/vitess/go/vt/tabletmanager/tmclient" "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "github.com/youtube/vitess/go/vt/topotools" "github.com/youtube/vitess/go/vt/topotools/events" "golang.org/x/net/context" @@ -60,7 +61,7 @@ func (agent *ActionAgent) TabletExternallyReparented(ctx context.Context, extern log.Warningf("fastTabletExternallyReparented: failed to read global shard record for %v/%v: %v", tablet.Keyspace, tablet.Shard, err) return err } - if topo.TabletAliasEqual(si.MasterAlias, tablet.Alias) { + if topoproto.TabletAliasEqual(si.MasterAlias, tablet.Alias) { // We may get called on the current master even when nothing has changed. // If the global shard record is already updated, it means we successfully // finished a previous reparent to this tablet. @@ -171,7 +172,7 @@ func (agent *ActionAgent) finalizeTabletExternallyReparented(ctx context.Context } }() - if !topo.TabletAliasIsZero(oldMasterAlias) { + if !topoproto.TabletAliasIsZero(oldMasterAlias) { wg.Add(1) go func() { // Force the old master to spare. diff --git a/go/vt/tabletmanager/rpc_server.go b/go/vt/tabletmanager/rpc_server.go index 14f9ea126f..b5a599ec96 100644 --- a/go/vt/tabletmanager/rpc_server.go +++ b/go/vt/tabletmanager/rpc_server.go @@ -11,7 +11,7 @@ import ( log "github.com/golang/glog" "github.com/youtube/vitess/go/tb" "github.com/youtube/vitess/go/vt/callinfo" - "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "golang.org/x/net/context" ) @@ -33,7 +33,7 @@ const rpcTimeout = time.Second * 30 func (agent *ActionAgent) rpcWrapper(ctx context.Context, name string, args, reply interface{}, verbose bool, f func() error, lock, runAfterAction bool) (err error) { defer func() { if x := recover(); x != nil { - log.Errorf("TabletManager.%v(%v) on %v panic: %v\n%s", name, args, topo.TabletAliasString(agent.TabletAlias), x, tb.Stack(4)) + log.Errorf("TabletManager.%v(%v) on %v panic: %v\n%s", name, args, topoproto.TabletAliasString(agent.TabletAlias), x, tb.Stack(4)) err = fmt.Errorf("caught panic during %v: %v", name, x) } }() @@ -54,11 +54,11 @@ func (agent *ActionAgent) rpcWrapper(ctx context.Context, name string, args, rep } if err = f(); err != nil { - log.Warningf("TabletManager.%v(%v)(on %v from %v) error: %v", name, args, topo.TabletAliasString(agent.TabletAlias), from, err.Error()) - return fmt.Errorf("TabletManager.%v on %v error: %v", name, topo.TabletAliasString(agent.TabletAlias), err) + log.Warningf("TabletManager.%v(%v)(on %v from %v) error: %v", name, args, topoproto.TabletAliasString(agent.TabletAlias), from, err.Error()) + return fmt.Errorf("TabletManager.%v on %v error: %v", name, topoproto.TabletAliasString(agent.TabletAlias), err) } if verbose { - log.Infof("TabletManager.%v(%v)(on %v from %v): %#v", name, args, topo.TabletAliasString(agent.TabletAlias), from, reply) + log.Infof("TabletManager.%v(%v)(on %v from %v): %#v", name, args, topoproto.TabletAliasString(agent.TabletAlias), from, reply) } if runAfterAction { err = agent.refreshTablet(ctx, "RPC("+name+")") diff --git a/go/vt/topo/events/tablet_change_syslog.go b/go/vt/topo/events/tablet_change_syslog.go index 4485a71c13..d6391ca8d1 100644 --- a/go/vt/topo/events/tablet_change_syslog.go +++ b/go/vt/topo/events/tablet_change_syslog.go @@ -5,13 +5,13 @@ import ( "log/syslog" "github.com/youtube/vitess/go/event/syslogger" - "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" ) // Syslog writes the event to syslog. func (tc *TabletChange) Syslog() (syslog.Priority, string) { return syslog.LOG_INFO, fmt.Sprintf("%s/%s/%s [tablet] %s", - tc.Tablet.Keyspace, tc.Tablet.Shard, topo.TabletAliasString(tc.Tablet.Alias), tc.Status) + tc.Tablet.Keyspace, tc.Tablet.Shard, topoproto.TabletAliasString(tc.Tablet.Alias), tc.Status) } var _ syslogger.Syslogger = (*TabletChange)(nil) // compile-time interface check diff --git a/go/vt/topo/proto3.go b/go/vt/topo/proto3.go index 94c3bd9b4f..99a4fa86fa 100644 --- a/go/vt/topo/proto3.go +++ b/go/vt/topo/proto3.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/youtube/vitess/go/vt/key" + "github.com/youtube/vitess/go/vt/topo/topoproto" pb "github.com/youtube/vitess/go/vt/proto/topodata" ) @@ -18,7 +19,7 @@ import ( // TabletTypeToProto turns a TabletType into a proto func TabletTypeToProto(t TabletType) pb.TabletType { - if result, err := ParseTabletType(string(t)); err != nil { + if result, err := topoproto.ParseTabletType(string(t)); err != nil { panic(fmt.Errorf("unknown tablet type: %v", t)) } else { return result diff --git a/go/vt/topo/replication.go b/go/vt/topo/replication.go index 1dc80a24db..b07ab5e768 100644 --- a/go/vt/topo/replication.go +++ b/go/vt/topo/replication.go @@ -10,6 +10,7 @@ import ( "github.com/youtube/vitess/go/trace" "github.com/youtube/vitess/go/vt/logutil" + "github.com/youtube/vitess/go/vt/topo/topoproto" pb "github.com/youtube/vitess/go/vt/proto/topodata" ) @@ -65,7 +66,7 @@ func UpdateShardReplicationRecord(ctx context.Context, ts Server, keyspace, shar span.StartClient("TopoServer.UpdateShardReplicationFields") span.Annotate("keyspace", keyspace) span.Annotate("shard", shard) - span.Annotate("tablet", TabletAliasString(tabletAlias)) + span.Annotate("tablet", topoproto.TabletAliasString(tabletAlias)) defer span.Finish() return ts.UpdateShardReplicationFields(ctx, tabletAlias.Cell, keyspace, shard, func(sr *pb.ShardReplication) error { diff --git a/go/vt/topo/shard.go b/go/vt/topo/shard.go index 1c952a0d56..231e0e320e 100644 --- a/go/vt/topo/shard.go +++ b/go/vt/topo/shard.go @@ -20,6 +20,7 @@ import ( "github.com/youtube/vitess/go/trace" "github.com/youtube/vitess/go/vt/concurrency" "github.com/youtube/vitess/go/vt/key" + "github.com/youtube/vitess/go/vt/topo/topoproto" pb "github.com/youtube/vitess/go/vt/proto/topodata" ) @@ -146,21 +147,6 @@ type ShardInfo struct { *pb.Shard } -// Keyspace returns the keyspace a shard belongs to -func (si *ShardInfo) Keyspace() string { - return si.keyspace -} - -// ShardName returns the shard name for a shard -func (si *ShardInfo) ShardName() string { - return si.shardName -} - -// Version returns the shard version from last time it was read or updated. -func (si *ShardInfo) Version() int64 { - return si.version -} - // NewShardInfo returns a ShardInfo basing on shard with the // keyspace / shard. This function should be only used by Server // implementations. @@ -173,6 +159,26 @@ func NewShardInfo(keyspace, shard string, value *pb.Shard, version int64) *Shard } } +// Keyspace returns the keyspace a shard belongs to. +func (si *ShardInfo) Keyspace() string { + return si.keyspace +} + +// ShardName returns the shard name for a shard. +func (si *ShardInfo) ShardName() string { + return si.shardName +} + +// Version returns the shard version from last time it was read or updated. +func (si *ShardInfo) Version() int64 { + return si.version +} + +// HasMaster returns true if the Shard has an assigned Master. +func (si *ShardInfo) HasMaster() bool { + return !topoproto.TabletAliasIsZero(si.Shard.MasterAlias) +} + // HasCell returns true if the cell is listed in the Cells for the shard. func (si *ShardInfo) HasCell(cell string) bool { for _, c := range si.Cells { @@ -271,7 +277,7 @@ func CreateShard(ctx context.Context, ts Server, keyspace, shard string) error { } } - for st, _ := range servedTypes { + for st := range servedTypes { s.ServedTypes = append(s.ServedTypes, &pb.Shard_ServedType{ TabletType: st, }) @@ -528,7 +534,7 @@ func FindAllTabletAliasesInShardByCell(ctx context.Context, ts Server, keyspace, } resultAsMap := make(map[pb.TabletAlias]bool) - if si.MasterAlias != nil && !TabletAliasIsZero(si.MasterAlias) { + if si.HasMaster() { if InCellList(si.MasterAlias.Cell, cells) { resultAsMap[*si.MasterAlias] = true } @@ -570,7 +576,7 @@ func FindAllTabletAliasesInShardByCell(ctx context.Context, ts Server, keyspace, v := a result = append(result, &v) } - sort.Sort(TabletAliasList(result)) + sort.Sort(topoproto.TabletAliasList(result)) return result, err } diff --git a/go/vt/topo/tablet.go b/go/vt/topo/tablet.go index c92eef7c87..8e2b8073dc 100644 --- a/go/vt/topo/tablet.go +++ b/go/vt/topo/tablet.go @@ -7,9 +7,6 @@ package topo import ( "fmt" "reflect" - "sort" - "strconv" - "strings" "sync" "golang.org/x/net/context" @@ -19,6 +16,7 @@ import ( "github.com/youtube/vitess/go/trace" pb "github.com/youtube/vitess/go/vt/proto/topodata" + "github.com/youtube/vitess/go/vt/topo/topoproto" ) const ( @@ -28,9 +26,6 @@ const ( // http://dev.mysql.com/doc/refman/5.1/en/replication-options.html NO_TABLET = 0 - // Default name for databases is the prefix plus keyspace - vtDbPrefix = "vt_" - // ReplicationLag is the key in the health map to indicate high // replication lag ReplicationLag = "replication_lag" @@ -40,92 +35,6 @@ const ( ReplicationLagHigh = "high" ) -// TabletAliasIsZero returns true iff cell and uid are empty -func TabletAliasIsZero(ta *pb.TabletAlias) bool { - return ta == nil || (ta.Cell == "" && ta.Uid == 0) -} - -// TabletAliasEqual returns true if two TabletAlias match -func TabletAliasEqual(left, right *pb.TabletAlias) bool { - if left == nil { - return right == nil - } - if right == nil { - return false - } - return *left == *right -} - -// TabletAliasString formats a TabletAlias -func TabletAliasString(tabletAlias *pb.TabletAlias) string { - if tabletAlias == nil { - return "" - } - return fmtAlias(tabletAlias.Cell, tabletAlias.Uid) -} - -// TabletAliasUIDStr returns a string version of the uid -func TabletAliasUIDStr(ta *pb.TabletAlias) string { - return tabletUIDStr(ta.Uid) -} - -// ParseTabletAliasString returns a TabletAlias for the input string, -// of the form - -func ParseTabletAliasString(aliasStr string) (*pb.TabletAlias, error) { - nameParts := strings.Split(aliasStr, "-") - if len(nameParts) != 2 { - return nil, fmt.Errorf("invalid tablet alias: %v", aliasStr) - } - uid, err := ParseUID(nameParts[1]) - if err != nil { - return nil, fmt.Errorf("invalid tablet uid %v: %v", aliasStr, err) - } - return &pb.TabletAlias{ - Cell: nameParts[0], - Uid: uid, - }, nil -} - -func tabletUIDStr(uid uint32) string { - return fmt.Sprintf("%010d", uid) -} - -// ParseUID parses just the uid (a number) -func ParseUID(value string) (uint32, error) { - uid, err := strconv.ParseUint(value, 10, 32) - if err != nil { - return 0, fmt.Errorf("bad tablet uid %v", err) - } - return uint32(uid), nil -} - -func fmtAlias(cell string, uid uint32) string { - return fmt.Sprintf("%v-%v", cell, tabletUIDStr(uid)) -} - -// TabletAliasList is used mainly for sorting -type TabletAliasList []*pb.TabletAlias - -// Len is part of sort.Interface -func (tal TabletAliasList) Len() int { - return len(tal) -} - -// Less is part of sort.Interface -func (tal TabletAliasList) Less(i, j int) bool { - if tal[i].Cell < tal[j].Cell { - return true - } else if tal[i].Cell > tal[j].Cell { - return false - } - return tal[i].Uid < tal[j].Uid -} - -// Swap is part of sort.Interface -func (tal TabletAliasList) Swap(i, j int) { - tal[i], tal[j] = tal[j], tal[i] -} - // TabletType is the main type for a tablet. It has an implication on: // - the replication graph // - the services run by vttablet on a tablet @@ -181,66 +90,6 @@ const ( TYPE_SCRAP = TabletType("scrap") ) -// AllTabletTypes lists all the possible tablet types -var AllTabletTypes = []pb.TabletType{ - pb.TabletType_IDLE, - pb.TabletType_MASTER, - pb.TabletType_REPLICA, - pb.TabletType_RDONLY, - pb.TabletType_BATCH, - pb.TabletType_SPARE, - pb.TabletType_EXPERIMENTAL, - pb.TabletType_SCHEMA_UPGRADE, - pb.TabletType_BACKUP, - pb.TabletType_RESTORE, - pb.TabletType_WORKER, - pb.TabletType_SCRAP, -} - -// SlaveTabletTypes contains all the tablet type that can have replication -// enabled. -var SlaveTabletTypes = []pb.TabletType{ - pb.TabletType_REPLICA, - pb.TabletType_RDONLY, - pb.TabletType_BATCH, - pb.TabletType_SPARE, - pb.TabletType_EXPERIMENTAL, - pb.TabletType_SCHEMA_UPGRADE, - pb.TabletType_BACKUP, - pb.TabletType_RESTORE, - pb.TabletType_WORKER, -} - -// ParseTabletType parses the tablet type into the enum -func ParseTabletType(param string) (pb.TabletType, error) { - value, ok := pb.TabletType_value[strings.ToUpper(param)] - if !ok { - return pb.TabletType_UNKNOWN, fmt.Errorf("unknown TabletType %v", param) - } - return pb.TabletType(value), nil -} - -// IsTypeInList returns true if the given type is in the list. -// Use it with AllTabletType and SlaveTabletType for instance. -func IsTypeInList(tabletType pb.TabletType, types []pb.TabletType) bool { - for _, t := range types { - if tabletType == t { - return true - } - } - return false -} - -// MakeStringTypeList returns a list of strings that match the input list. -func MakeStringTypeList(types []pb.TabletType) []string { - strs := make([]string, len(types)) - for i, t := range types { - strs[i] = strings.ToLower(t.String()) - } - sort.Strings(strs) - return strs -} - // IsTrivialTypeChange returns if this db type be trivially reassigned // without changes to the replication graph func IsTrivialTypeChange(oldTabletType, newTabletType pb.TabletType) bool { @@ -346,23 +195,6 @@ func TabletEndPoint(tablet *pb.Tablet) (*pb.EndPoint, error) { return entry, nil } -// TabletAddr returns hostname:vt port associated with a tablet -func TabletAddr(tablet *pb.Tablet) string { - return netutil.JoinHostPort(tablet.Hostname, tablet.PortMap["vt"]) -} - -// TabletDbName is usually implied by keyspace. Having the shard information in the -// database name complicates mysql replication. -func TabletDbName(tablet *pb.Tablet) string { - if tablet.DbNameOverride != "" { - return tablet.DbNameOverride - } - if tablet.Keyspace == "" { - return "" - } - return vtDbPrefix + tablet.Keyspace -} - // TabletComplete validates and normalizes the tablet. If the shard name // contains a '-' it is going to try to infer the keyrange from it. func TabletComplete(tablet *pb.Tablet) error { @@ -383,12 +215,12 @@ type TabletInfo struct { // String returns a string describing the tablet. func (ti *TabletInfo) String() string { - return fmt.Sprintf("Tablet{%v}", TabletAliasString(ti.Alias)) + return fmt.Sprintf("Tablet{%v}", topoproto.TabletAliasString(ti.Alias)) } // AliasString returns the string representation of the tablet alias func (ti *TabletInfo) AliasString() string { - return TabletAliasString(ti.Alias) + return topoproto.TabletAliasString(ti.Alias) } // Addr returns hostname:vt port. @@ -411,7 +243,7 @@ func (ti *TabletInfo) IsAssigned() bool { // DbName is usually implied by keyspace. Having the shard information in the // database name complicates mysql replication. func (ti *TabletInfo) DbName() string { - return TabletDbName(ti.Tablet) + return topoproto.TabletDbName(ti.Tablet) } // Version returns the version of this tablet from last time it was read or @@ -457,7 +289,7 @@ func NewTabletInfo(tablet *pb.Tablet, version int64) *TabletInfo { func GetTablet(ctx context.Context, ts Server, alias *pb.TabletAlias) (*TabletInfo, error) { span := trace.NewSpanFromContext(ctx) span.StartClient("TopoServer.GetTablet") - span.Annotate("tablet", TabletAliasString(alias)) + span.Annotate("tablet", topoproto.TabletAliasString(alias)) defer span.Finish() return ts.GetTablet(ctx, alias) @@ -467,7 +299,7 @@ func GetTablet(ctx context.Context, ts Server, alias *pb.TabletAlias) (*TabletIn func UpdateTablet(ctx context.Context, ts Server, tablet *TabletInfo) error { span := trace.NewSpanFromContext(ctx) span.StartClient("TopoServer.UpdateTablet") - span.Annotate("tablet", TabletAliasString(tablet.Alias)) + span.Annotate("tablet", topoproto.TabletAliasString(tablet.Alias)) defer span.Finish() var version int64 = -1 @@ -487,7 +319,7 @@ func UpdateTablet(ctx context.Context, ts Server, tablet *TabletInfo) error { func UpdateTabletFields(ctx context.Context, ts Server, alias *pb.TabletAlias, update func(*pb.Tablet) error) error { span := trace.NewSpanFromContext(ctx) span.StartClient("TopoServer.UpdateTabletFields") - span.Annotate("tablet", TabletAliasString(alias)) + span.Annotate("tablet", topoproto.TabletAliasString(alias)) defer span.Finish() return ts.UpdateTabletFields(ctx, alias, update) @@ -500,8 +332,8 @@ func Validate(ctx context.Context, ts Server, tabletAlias *pb.TabletAlias) error if err != nil { return err } - if !TabletAliasEqual(tablet.Alias, tabletAlias) { - return fmt.Errorf("bad tablet alias data for tablet %v: %#v", TabletAliasString(tabletAlias), tablet.Alias) + if !topoproto.TabletAliasEqual(tablet.Alias, tabletAlias) { + return fmt.Errorf("bad tablet alias data for tablet %v: %#v", topoproto.TabletAliasString(tabletAlias), tablet.Alias) } // Some tablets have no information to generate valid replication paths. diff --git a/go/vt/topo/topoproto/tablet.go b/go/vt/topo/topoproto/tablet.go new file mode 100644 index 0000000000..b65e9a1079 --- /dev/null +++ b/go/vt/topo/topoproto/tablet.go @@ -0,0 +1,180 @@ +// Copyright 2015, Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package topoproto contains utility functions to deal with the proto3 +// structures defined in proto/topodata. +package topoproto + +import ( + "fmt" + "sort" + "strconv" + "strings" + + "github.com/youtube/vitess/go/netutil" + + pb "github.com/youtube/vitess/go/vt/proto/topodata" +) + +// This file contains the topodata.Tablet utility functions. + +const ( + // Default name for databases is the prefix plus keyspace + vtDbPrefix = "vt_" +) + +// TabletAliasIsZero returns true iff cell and uid are empty +func TabletAliasIsZero(ta *pb.TabletAlias) bool { + return ta == nil || (ta.Cell == "" && ta.Uid == 0) +} + +// TabletAliasEqual returns true if two TabletAlias match +func TabletAliasEqual(left, right *pb.TabletAlias) bool { + if left == nil { + return right == nil + } + if right == nil { + return false + } + return *left == *right +} + +// TabletAliasString formats a TabletAlias +func TabletAliasString(ta *pb.TabletAlias) string { + if ta == nil { + return "" + } + return fmt.Sprintf("%v-%010d", ta.Cell, ta.Uid) +} + +// TabletAliasUIDStr returns a string version of the uid +func TabletAliasUIDStr(ta *pb.TabletAlias) string { + return fmt.Sprintf("%010d", ta.Uid) +} + +// ParseTabletAlias returns a TabletAlias for the input string, +// of the form - +func ParseTabletAlias(aliasStr string) (*pb.TabletAlias, error) { + nameParts := strings.Split(aliasStr, "-") + if len(nameParts) != 2 { + return nil, fmt.Errorf("invalid tablet alias: %v", aliasStr) + } + uid, err := ParseUID(nameParts[1]) + if err != nil { + return nil, fmt.Errorf("invalid tablet uid %v: %v", aliasStr, err) + } + return &pb.TabletAlias{ + Cell: nameParts[0], + Uid: uid, + }, nil +} + +// ParseUID parses just the uid (a number) +func ParseUID(value string) (uint32, error) { + uid, err := strconv.ParseUint(value, 10, 32) + if err != nil { + return 0, fmt.Errorf("bad tablet uid %v", err) + } + return uint32(uid), nil +} + +// TabletAliasList is used mainly for sorting +type TabletAliasList []*pb.TabletAlias + +// Len is part of sort.Interface +func (tal TabletAliasList) Len() int { + return len(tal) +} + +// Less is part of sort.Interface +func (tal TabletAliasList) Less(i, j int) bool { + if tal[i].Cell < tal[j].Cell { + return true + } else if tal[i].Cell > tal[j].Cell { + return false + } + return tal[i].Uid < tal[j].Uid +} + +// Swap is part of sort.Interface +func (tal TabletAliasList) Swap(i, j int) { + tal[i], tal[j] = tal[j], tal[i] +} + +// AllTabletTypes lists all the possible tablet types +var AllTabletTypes = []pb.TabletType{ + pb.TabletType_IDLE, + pb.TabletType_MASTER, + pb.TabletType_REPLICA, + pb.TabletType_RDONLY, + pb.TabletType_BATCH, + pb.TabletType_SPARE, + pb.TabletType_EXPERIMENTAL, + pb.TabletType_SCHEMA_UPGRADE, + pb.TabletType_BACKUP, + pb.TabletType_RESTORE, + pb.TabletType_WORKER, + pb.TabletType_SCRAP, +} + +// SlaveTabletTypes contains all the tablet type that can have replication +// enabled. +var SlaveTabletTypes = []pb.TabletType{ + pb.TabletType_REPLICA, + pb.TabletType_RDONLY, + pb.TabletType_BATCH, + pb.TabletType_SPARE, + pb.TabletType_EXPERIMENTAL, + pb.TabletType_SCHEMA_UPGRADE, + pb.TabletType_BACKUP, + pb.TabletType_RESTORE, + pb.TabletType_WORKER, +} + +// ParseTabletType parses the tablet type into the enum +func ParseTabletType(param string) (pb.TabletType, error) { + value, ok := pb.TabletType_value[strings.ToUpper(param)] + if !ok { + return pb.TabletType_UNKNOWN, fmt.Errorf("unknown TabletType %v", param) + } + return pb.TabletType(value), nil +} + +// IsTypeInList returns true if the given type is in the list. +// Use it with AllTabletType and SlaveTabletType for instance. +func IsTypeInList(tabletType pb.TabletType, types []pb.TabletType) bool { + for _, t := range types { + if tabletType == t { + return true + } + } + return false +} + +// MakeStringTypeList returns a list of strings that match the input list. +func MakeStringTypeList(types []pb.TabletType) []string { + strs := make([]string, len(types)) + for i, t := range types { + strs[i] = strings.ToLower(t.String()) + } + sort.Strings(strs) + return strs +} + +// TabletAddr returns hostname:vt port associated with a tablet +func TabletAddr(tablet *pb.Tablet) string { + return netutil.JoinHostPort(tablet.Hostname, tablet.PortMap["vt"]) +} + +// TabletDbName is usually implied by keyspace. Having the shard +// information in the database name complicates mysql replication. +func TabletDbName(tablet *pb.Tablet) string { + if tablet.DbNameOverride != "" { + return tablet.DbNameOverride + } + if tablet.Keyspace == "" { + return "" + } + return vtDbPrefix + tablet.Keyspace +} diff --git a/go/vt/topotools/events/reparent_syslog.go b/go/vt/topotools/events/reparent_syslog.go index 9551a919f1..4f8991f60b 100644 --- a/go/vt/topotools/events/reparent_syslog.go +++ b/go/vt/topotools/events/reparent_syslog.go @@ -9,15 +9,15 @@ import ( "log/syslog" "github.com/youtube/vitess/go/event/syslogger" - "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" ) // Syslog writes a Reparent event to syslog. func (r *Reparent) Syslog() (syslog.Priority, string) { return syslog.LOG_INFO, fmt.Sprintf("%s/%s [reparent %v -> %v] %s (%s)", r.ShardInfo.Keyspace(), r.ShardInfo.ShardName(), - topo.TabletAliasString(r.OldMaster.Alias), - topo.TabletAliasString(r.NewMaster.Alias), + topoproto.TabletAliasString(r.OldMaster.Alias), + topoproto.TabletAliasString(r.NewMaster.Alias), r.Status, r.ExternalID) } diff --git a/go/vt/topotools/tablet.go b/go/vt/topotools/tablet.go index ad71a9f713..b82b367d22 100644 --- a/go/vt/topotools/tablet.go +++ b/go/vt/topotools/tablet.go @@ -29,6 +29,7 @@ import ( log "github.com/golang/glog" "github.com/youtube/vitess/go/vt/hook" "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" pb "github.com/youtube/vitess/go/vt/proto/topodata" ) @@ -39,7 +40,7 @@ func ConfigureTabletHook(hk *hook.Hook, tabletAlias *pb.TabletAlias) { if hk.ExtraEnv == nil { hk.ExtraEnv = make(map[string]string, 1) } - hk.ExtraEnv["TABLET_ALIAS"] = topo.TabletAliasString(tabletAlias) + hk.ExtraEnv["TABLET_ALIAS"] = topoproto.TabletAliasString(tabletAlias) } // Scrap will update the tablet type to 'Scrap', and remove it from diff --git a/go/vt/topotools/topology.go b/go/vt/topotools/topology.go index d847244d18..8b35fadbbd 100644 --- a/go/vt/topotools/topology.go +++ b/go/vt/topotools/topology.go @@ -14,6 +14,7 @@ import ( "github.com/youtube/vitess/go/netutil" "github.com/youtube/vitess/go/vt/concurrency" "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" pb "github.com/youtube/vitess/go/vt/proto/topodata" ) @@ -147,7 +148,7 @@ func (ks *KeyspaceNodes) hasOnlyNumericShardNames() bool { // contains. func (ks KeyspaceNodes) TabletTypes() []pb.TabletType { var contained []pb.TabletType - for _, t := range topo.AllTabletTypes { + for _, t := range topoproto.AllTabletTypes { if ks.HasType(t) { contained = append(contained, t) } diff --git a/go/vt/topotools/topology_test.go b/go/vt/topotools/topology_test.go index 4121fbb7c1..8a926ee815 100644 --- a/go/vt/topotools/topology_test.go +++ b/go/vt/topotools/topology_test.go @@ -9,9 +9,8 @@ import ( "sort" "testing" - "github.com/youtube/vitess/go/vt/topo" - pb "github.com/youtube/vitess/go/vt/proto/topodata" + "github.com/youtube/vitess/go/vt/topo/topoproto" ) func TestTabletNodeShortName(t *testing.T) { @@ -97,8 +96,8 @@ func TestKeyspaceNodesTabletTypes(t *testing.T) { }, }, } - want := topo.MakeStringTypeList([]pb.TabletType{pb.TabletType_REPLICA, pb.TabletType_MASTER}) - got := topo.MakeStringTypeList(input.TabletTypes()) + want := topoproto.MakeStringTypeList([]pb.TabletType{pb.TabletType_REPLICA, pb.TabletType_MASTER}) + got := topoproto.MakeStringTypeList(input.TabletTypes()) if !reflect.DeepEqual(got, want) { t.Errorf("KeyspaceNodes.TabletTypes() = %v, want %v", got, want) } diff --git a/go/vt/topotools/utils.go b/go/vt/topotools/utils.go index f407df4882..b63fc24719 100644 --- a/go/vt/topotools/utils.go +++ b/go/vt/topotools/utils.go @@ -13,6 +13,7 @@ import ( log "github.com/golang/glog" "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" pb "github.com/youtube/vitess/go/vt/proto/topodata" ) @@ -33,7 +34,7 @@ func GetAllTablets(ctx context.Context, ts topo.Server, cell string) ([]*topo.Ta if err != nil { return nil, err } - sort.Sort(topo.TabletAliasList(aliases)) + sort.Sort(topoproto.TabletAliasList(aliases)) tabletMap, err := topo.GetTabletMap(ctx, ts, aliases) if err != nil { diff --git a/go/vt/vtctl/query.go b/go/vt/vtctl/query.go index 164421490e..66139c7fae 100644 --- a/go/vt/vtctl/query.go +++ b/go/vt/vtctl/query.go @@ -14,6 +14,7 @@ import ( "github.com/youtube/vitess/go/vt/tabletserver/tabletconn" "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "github.com/youtube/vitess/go/vt/vtgate/vtgateconn" "github.com/youtube/vitess/go/vt/wrangler" "golang.org/x/net/context" @@ -214,11 +215,11 @@ func commandVtTabletExecute(ctx context.Context, wr *wrangler.Wrangler, subFlags if subFlags.NArg() != 2 { return fmt.Errorf("the and arguments are required for the VtTabletExecute command") } - tt, err := topo.ParseTabletType(*tabletType) + tt, err := topoproto.ParseTabletType(*tabletType) if err != nil { return err } - tabletAlias, err := topo.ParseTabletAliasString(subFlags.Arg(0)) + tabletAlias, err := topoproto.ParseTabletAlias(subFlags.Arg(0)) if err != nil { return err } @@ -255,11 +256,11 @@ func commandVtTabletBegin(ctx context.Context, wr *wrangler.Wrangler, subFlags * if subFlags.NArg() != 1 { return fmt.Errorf("the argument is required for the VtTabletBegin command") } - tt, err := topo.ParseTabletType(*tabletType) + tt, err := topoproto.ParseTabletType(*tabletType) if err != nil { return err } - tabletAlias, err := topo.ParseTabletAliasString(subFlags.Arg(0)) + tabletAlias, err := topoproto.ParseTabletAlias(subFlags.Arg(0)) if err != nil { return err } @@ -303,11 +304,11 @@ func commandVtTabletCommit(ctx context.Context, wr *wrangler.Wrangler, subFlags if err != nil { return err } - tt, err := topo.ParseTabletType(*tabletType) + tt, err := topoproto.ParseTabletType(*tabletType) if err != nil { return err } - tabletAlias, err := topo.ParseTabletAliasString(subFlags.Arg(0)) + tabletAlias, err := topoproto.ParseTabletAlias(subFlags.Arg(0)) if err != nil { return err } @@ -344,11 +345,11 @@ func commandVtTabletRollback(ctx context.Context, wr *wrangler.Wrangler, subFlag if err != nil { return err } - tt, err := topo.ParseTabletType(*tabletType) + tt, err := topoproto.ParseTabletType(*tabletType) if err != nil { return err } - tabletAlias, err := topo.ParseTabletAliasString(subFlags.Arg(0)) + tabletAlias, err := topoproto.ParseTabletAlias(subFlags.Arg(0)) if err != nil { return err } @@ -379,7 +380,7 @@ func commandVtTabletStreamHealth(ctx context.Context, wr *wrangler.Wrangler, sub if subFlags.NArg() != 1 { return fmt.Errorf("The argument is required for the VtTabletStreamHealth command.") } - tabletAlias, err := topo.ParseTabletAliasString(subFlags.Arg(0)) + tabletAlias, err := topoproto.ParseTabletAlias(subFlags.Arg(0)) if err != nil { return err } diff --git a/go/vt/vtctl/reparent.go b/go/vt/vtctl/reparent.go index d488a51c02..8c94bd7bcb 100644 --- a/go/vt/vtctl/reparent.go +++ b/go/vt/vtctl/reparent.go @@ -10,6 +10,7 @@ import ( "time" "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "github.com/youtube/vitess/go/vt/wrangler" "golang.org/x/net/context" ) @@ -57,7 +58,7 @@ func commandDemoteMaster(ctx context.Context, wr *wrangler.Wrangler, subFlags *f if subFlags.NArg() != 1 { return fmt.Errorf("action DemoteMaster requires ") } - tabletAlias, err := topo.ParseTabletAliasString(subFlags.Arg(0)) + tabletAlias, err := topoproto.ParseTabletAlias(subFlags.Arg(0)) if err != nil { return err } @@ -80,7 +81,7 @@ func commandReparentTablet(ctx context.Context, wr *wrangler.Wrangler, subFlags if subFlags.NArg() != 1 { return fmt.Errorf("action ReparentTablet requires ") } - tabletAlias, err := topo.ParseTabletAliasString(subFlags.Arg(0)) + tabletAlias, err := topoproto.ParseTabletAlias(subFlags.Arg(0)) if err != nil { return err } @@ -104,7 +105,7 @@ func commandInitShardMaster(ctx context.Context, wr *wrangler.Wrangler, subFlags if err != nil { return err } - tabletAlias, err := topo.ParseTabletAliasString(subFlags.Arg(1)) + tabletAlias, err := topoproto.ParseTabletAlias(subFlags.Arg(1)) if err != nil { return err } @@ -128,7 +129,7 @@ func commandPlannedReparentShard(ctx context.Context, wr *wrangler.Wrangler, sub if err != nil { return err } - tabletAlias, err := topo.ParseTabletAliasString(subFlags.Arg(1)) + tabletAlias, err := topoproto.ParseTabletAlias(subFlags.Arg(1)) if err != nil { return err } @@ -152,7 +153,7 @@ func commandEmergencyReparentShard(ctx context.Context, wr *wrangler.Wrangler, s if err != nil { return err } - tabletAlias, err := topo.ParseTabletAliasString(subFlags.Arg(1)) + tabletAlias, err := topoproto.ParseTabletAlias(subFlags.Arg(1)) if err != nil { return err } diff --git a/go/vt/vtctl/vtctl.go b/go/vt/vtctl/vtctl.go index 82e1ebb48b..8ba7d2a229 100644 --- a/go/vt/vtctl/vtctl.go +++ b/go/vt/vtctl/vtctl.go @@ -107,6 +107,7 @@ import ( "github.com/youtube/vitess/go/vt/tabletmanager/actionnode" "github.com/youtube/vitess/go/vt/tabletserver/tabletconn" "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "github.com/youtube/vitess/go/vt/topotools" "github.com/youtube/vitess/go/vt/wrangler" ) @@ -135,7 +136,7 @@ var commands = []commandGroup{ "[-force] [-parent] [-update] [-db-name-override=] [-hostname=] [-mysql_port=] [-port=] [-grpc_port=] [-keyspace=] [-shard=] [-parent_alias=] ", "Initializes a tablet in the topology.\n" + "Valid values are:\n" + - " " + strings.Join(topo.MakeStringTypeList(topo.AllTabletTypes), " ")}, + " " + strings.Join(topoproto.MakeStringTypeList(topoproto.AllTabletTypes), " ")}, command{"GetTablet", commandGetTablet, "", "Outputs a JSON structure that contains information about the Tablet."}, @@ -165,7 +166,7 @@ var commands = []commandGroup{ "Changes the db type for the specified tablet, if possible. This command is used primarily to arrange replicas, and it will not convert a master.\n" + "NOTE: This command automatically updates the serving graph.\n" + "Valid values are:\n" + - " " + strings.Join(topo.MakeStringTypeList(topo.SlaveTabletTypes), " ")}, + " " + strings.Join(topoproto.MakeStringTypeList(topoproto.SlaveTabletTypes), " ")}, command{"Ping", commandPing, "", "Checks that the specified tablet is awake and responding to RPCs. This command can be blocked by other in-flight operations."}, @@ -421,7 +422,7 @@ func fmtTabletAwkable(ti *topo.TabletInfo) string { if shard == "" { shard = "" } - return fmt.Sprintf("%v %v %v %v %v %v %v", topo.TabletAliasString(ti.Alias), keyspace, shard, strings.ToLower(ti.Type.String()), ti.Addr(), ti.MysqlAddr(), fmtMapAwkable(ti.Tags)) + return fmt.Sprintf("%v %v %v %v %v %v %v", topoproto.TabletAliasString(ti.Alias), keyspace, shard, strings.ToLower(ti.Type.String()), ti.Addr(), ti.MysqlAddr(), fmtMapAwkable(ti.Tags)) } func fmtAction(action *actionnode.ActionNode) string { @@ -550,7 +551,7 @@ func tabletParamsToTabletAliases(params []string) ([]*pb.TabletAlias, error) { result := make([]*pb.TabletAlias, len(params)) var err error for i, param := range params { - result[i], err = topo.ParseTabletAliasString(param) + result[i], err = topoproto.ParseTabletAlias(param) if err != nil { return nil, err } @@ -561,12 +562,12 @@ func tabletParamsToTabletAliases(params []string) ([]*pb.TabletAlias, error) { // parseTabletType parses the string tablet type and verifies // it is an accepted one func parseTabletType(param string, types []pb.TabletType) (pb.TabletType, error) { - tabletType, err := topo.ParseTabletType(param) + tabletType, err := topoproto.ParseTabletType(param) if err != nil { return pb.TabletType_UNKNOWN, fmt.Errorf("invalid tablet type %v: %v", param, err) } - if !topo.IsTypeInList(pb.TabletType(tabletType), types) { - return pb.TabletType_UNKNOWN, fmt.Errorf("Type %v is not one of: %v", tabletType, strings.Join(topo.MakeStringTypeList(types), " ")) + if !topoproto.IsTypeInList(pb.TabletType(tabletType), types) { + return pb.TabletType_UNKNOWN, fmt.Errorf("Type %v is not one of: %v", tabletType, strings.Join(topoproto.MakeStringTypeList(types), " ")) } return tabletType, nil } @@ -586,7 +587,7 @@ func parseKeyspaceIDType(param string) (pb.KeyspaceIdType, error) { // parseServingTabletType3 parses the tablet type into the enum, // and makes sure the enum is of serving type (MASTER, REPLICA, RDONLY/BATCH) func parseServingTabletType3(param string) (pb.TabletType, error) { - servedType, err := topo.ParseTabletType(param) + servedType, err := topoproto.ParseTabletType(param) if err != nil { return pb.TabletType_UNKNOWN, err } @@ -618,11 +619,11 @@ func commandInitTablet(ctx context.Context, wr *wrangler.Wrangler, subFlags *fla if subFlags.NArg() != 2 { return fmt.Errorf("The and arguments are both required for the InitTablet command.") } - tabletAlias, err := topo.ParseTabletAliasString(subFlags.Arg(0)) + tabletAlias, err := topoproto.ParseTabletAlias(subFlags.Arg(0)) if err != nil { return err } - tabletType, err := parseTabletType(subFlags.Arg(1), topo.AllTabletTypes) + tabletType, err := parseTabletType(subFlags.Arg(1), topoproto.AllTabletTypes) if err != nil { return err } @@ -659,7 +660,7 @@ func commandGetTablet(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag return fmt.Errorf("The argument is required for the GetTablet command.") } - tabletAlias, err := topo.ParseTabletAliasString(subFlags.Arg(0)) + tabletAlias, err := topoproto.ParseTabletAlias(subFlags.Arg(0)) if err != nil { return err } @@ -687,7 +688,7 @@ func commandUpdateTabletAddrs(ctx context.Context, wr *wrangler.Wrangler, subFla return fmt.Errorf("malformed address: %v", *ipAddr) } - tabletAlias, err := topo.ParseTabletAliasString(subFlags.Arg(0)) + tabletAlias, err := topoproto.ParseTabletAlias(subFlags.Arg(0)) if err != nil { return err } @@ -726,7 +727,7 @@ func commandScrapTablet(ctx context.Context, wr *wrangler.Wrangler, subFlags *fl return fmt.Errorf("The argument is required for the ScrapTablet command.") } - tabletAlias, err := topo.ParseTabletAliasString(subFlags.Arg(0)) + tabletAlias, err := topoproto.ParseTabletAlias(subFlags.Arg(0)) if err != nil { return err } @@ -761,7 +762,7 @@ func commandSetReadOnly(ctx context.Context, wr *wrangler.Wrangler, subFlags *fl return fmt.Errorf("The argument is required for the SetReadOnly command.") } - tabletAlias, err := topo.ParseTabletAliasString(subFlags.Arg(0)) + tabletAlias, err := topoproto.ParseTabletAlias(subFlags.Arg(0)) if err != nil { return err } @@ -780,7 +781,7 @@ func commandSetReadWrite(ctx context.Context, wr *wrangler.Wrangler, subFlags *f return fmt.Errorf("The argument is required for the SetReadWrite command.") } - tabletAlias, err := topo.ParseTabletAliasString(subFlags.Arg(0)) + tabletAlias, err := topoproto.ParseTabletAlias(subFlags.Arg(0)) if err != nil { return err } @@ -799,7 +800,7 @@ func commandStartSlave(ctx context.Context, wr *wrangler.Wrangler, subFlags *fla return fmt.Errorf("action StartSlave requires ") } - tabletAlias, err := topo.ParseTabletAliasString(subFlags.Arg(0)) + tabletAlias, err := topoproto.ParseTabletAlias(subFlags.Arg(0)) if err != nil { return err } @@ -818,7 +819,7 @@ func commandStopSlave(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag return fmt.Errorf("action StopSlave requires ") } - tabletAlias, err := topo.ParseTabletAliasString(subFlags.Arg(0)) + tabletAlias, err := topoproto.ParseTabletAlias(subFlags.Arg(0)) if err != nil { return err } @@ -840,11 +841,11 @@ func commandChangeSlaveType(ctx context.Context, wr *wrangler.Wrangler, subFlags return fmt.Errorf("The and arguments are required for the ChangeSlaveType command.") } - tabletAlias, err := topo.ParseTabletAliasString(subFlags.Arg(0)) + tabletAlias, err := topoproto.ParseTabletAlias(subFlags.Arg(0)) if err != nil { return err } - newType, err := parseTabletType(subFlags.Arg(1), topo.AllTabletTypes) + newType, err := parseTabletType(subFlags.Arg(1), topoproto.AllTabletTypes) if err != nil { return err } @@ -871,7 +872,7 @@ func commandPing(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.Flag if subFlags.NArg() != 1 { return fmt.Errorf("The argument is required for the Ping command.") } - tabletAlias, err := topo.ParseTabletAliasString(subFlags.Arg(0)) + tabletAlias, err := topoproto.ParseTabletAlias(subFlags.Arg(0)) if err != nil { return err } @@ -889,7 +890,7 @@ func commandRefreshState(ctx context.Context, wr *wrangler.Wrangler, subFlags *f if subFlags.NArg() != 1 { return fmt.Errorf("The argument is required for the RefreshState command.") } - tabletAlias, err := topo.ParseTabletAliasString(subFlags.Arg(0)) + tabletAlias, err := topoproto.ParseTabletAlias(subFlags.Arg(0)) if err != nil { return err } @@ -907,7 +908,7 @@ func commandRunHealthCheck(ctx context.Context, wr *wrangler.Wrangler, subFlags if subFlags.NArg() != 2 { return fmt.Errorf("The and arguments are required for the RunHealthCheck command.") } - tabletAlias, err := topo.ParseTabletAliasString(subFlags.Arg(0)) + tabletAlias, err := topoproto.ParseTabletAlias(subFlags.Arg(0)) if err != nil { return err } @@ -929,7 +930,7 @@ func commandSleep(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.Fla if subFlags.NArg() != 2 { return fmt.Errorf("The and arguments are required for the Sleep command.") } - tabletAlias, err := topo.ParseTabletAliasString(subFlags.Arg(0)) + tabletAlias, err := topoproto.ParseTabletAlias(subFlags.Arg(0)) if err != nil { return err } @@ -953,7 +954,7 @@ func commandBackup(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.Fl return fmt.Errorf("The Backup command requires the argument.") } - tabletAlias, err := topo.ParseTabletAliasString(subFlags.Arg(0)) + tabletAlias, err := topoproto.ParseTabletAlias(subFlags.Arg(0)) if err != nil { return err } @@ -984,7 +985,7 @@ func commandExecuteFetchAsDba(ctx context.Context, wr *wrangler.Wrangler, subFla return fmt.Errorf("The and arguments are required for the ExecuteFetchAsDba command.") } - alias, err := topo.ParseTabletAliasString(subFlags.Arg(0)) + alias, err := topoproto.ParseTabletAlias(subFlags.Arg(0)) if err != nil { return err } @@ -1004,7 +1005,7 @@ func commandExecuteHook(ctx context.Context, wr *wrangler.Wrangler, subFlags *fl return fmt.Errorf("The and arguments are required for the ExecuteHook command.") } - tabletAlias, err := topo.ParseTabletAliasString(subFlags.Arg(0)) + tabletAlias, err := topoproto.ParseTabletAlias(subFlags.Arg(0)) if err != nil { return err } @@ -1097,7 +1098,7 @@ func commandTabletExternallyReparented(ctx context.Context, wr *wrangler.Wrangle return fmt.Errorf("The argument is required for the TabletExternallyReparented command.") } - tabletAlias, err := topo.ParseTabletAliasString(subFlags.Arg(0)) + tabletAlias, err := topoproto.ParseTabletAlias(subFlags.Arg(0)) if err != nil { return err } @@ -1292,7 +1293,7 @@ func commandShardReplicationAdd(ctx context.Context, wr *wrangler.Wrangler, subF if err != nil { return err } - tabletAlias, err := topo.ParseTabletAliasString(subFlags.Arg(1)) + tabletAlias, err := topoproto.ParseTabletAlias(subFlags.Arg(1)) if err != nil { return err } @@ -1311,7 +1312,7 @@ func commandShardReplicationRemove(ctx context.Context, wr *wrangler.Wrangler, s if err != nil { return err } - tabletAlias, err := topo.ParseTabletAliasString(subFlags.Arg(1)) + tabletAlias, err := topoproto.ParseTabletAlias(subFlags.Arg(1)) if err != nil { return err } @@ -1356,10 +1357,10 @@ func commandWaitForFilteredReplication(ctx context.Context, wr *wrangler.Wrangle if len(shardInfo.SourceShards) == 0 { return fmt.Errorf("shard %v/%v has no source shard", keyspace, shard) } - alias := shardInfo.MasterAlias - if topo.TabletAliasIsZero(alias) { + if !shardInfo.HasMaster() { return fmt.Errorf("shard %v/%v has no master", keyspace, shard) } + alias := shardInfo.MasterAlias tabletInfo, err := wr.TopoServer().GetTablet(ctx, alias) if err != nil { return err @@ -1722,7 +1723,7 @@ func commandResolve(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.F return fmt.Errorf("The Resolve command requires a single argument, the value of which must be in the format ..:.") } - tabletType, err := parseTabletType(parts[2], topo.AllTabletTypes) + tabletType, err := parseTabletType(parts[2], topoproto.AllTabletTypes) if err != nil { return err } @@ -1790,7 +1791,7 @@ func commandListTablets(ctx context.Context, wr *wrangler.Wrangler, subFlags *fl aliases := make([]*pb.TabletAlias, len(paths)) var err error for i, path := range paths { - aliases[i], err = topo.ParseTabletAliasString(path) + aliases[i], err = topoproto.ParseTabletAlias(path) if err != nil { return err } @@ -1809,7 +1810,7 @@ func commandGetSchema(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag if subFlags.NArg() != 1 { return fmt.Errorf("The argument is required for the GetSchema command.") } - tabletAlias, err := topo.ParseTabletAliasString(subFlags.Arg(0)) + tabletAlias, err := topoproto.ParseTabletAlias(subFlags.Arg(0)) if err != nil { return err } @@ -1842,7 +1843,7 @@ func commandReloadSchema(ctx context.Context, wr *wrangler.Wrangler, subFlags *f if subFlags.NArg() != 1 { return fmt.Errorf("The argument is required for the ReloadSchema command.") } - tabletAlias, err := topo.ParseTabletAliasString(subFlags.Arg(0)) + tabletAlias, err := topoproto.ParseTabletAlias(subFlags.Arg(0)) if err != nil { return err } @@ -1940,7 +1941,7 @@ func commandCopySchemaShard(ctx context.Context, wr *wrangler.Wrangler, subFlags if err == nil { return wr.CopySchemaShardFromShard(ctx, tableArray, excludeTableArray, *includeViews, sourceKeyspace, sourceShard, destKeyspace, destShard) } - sourceTabletAlias, err := topo.ParseTabletAliasString(subFlags.Arg(0)) + sourceTabletAlias, err := topoproto.ParseTabletAlias(subFlags.Arg(0)) if err == nil { return wr.CopySchemaShard(ctx, sourceTabletAlias, tableArray, excludeTableArray, *includeViews, destKeyspace, destShard) } @@ -1981,7 +1982,7 @@ func commandGetPermissions(ctx context.Context, wr *wrangler.Wrangler, subFlags if subFlags.NArg() != 1 { return fmt.Errorf("The argument is required for the GetPermissions command.") } - tabletAlias, err := topo.ParseTabletAliasString(subFlags.Arg(0)) + tabletAlias, err := topoproto.ParseTabletAlias(subFlags.Arg(0)) if err != nil { return err } diff --git a/go/vt/worker/clone_utils.go b/go/vt/worker/clone_utils.go index f56a962ced..bab36a4306 100644 --- a/go/vt/worker/clone_utils.go +++ b/go/vt/worker/clone_utils.go @@ -20,6 +20,7 @@ import ( "github.com/youtube/vitess/go/sqltypes" myproto "github.com/youtube/vitess/go/vt/mysqlctl/proto" "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "github.com/youtube/vitess/go/vt/wrangler" pb "github.com/youtube/vitess/go/vt/proto/topodata" @@ -39,18 +40,18 @@ func resolveDestinationShardMaster(ctx context.Context, keyspace, shard string, return ti, fmt.Errorf("unable to resolve destination shard %v/%v", keyspace, shard) } - if topo.TabletAliasIsZero(si.MasterAlias) { + if !si.HasMaster() { return ti, fmt.Errorf("no master in destination shard %v/%v", keyspace, shard) } - wr.Logger().Infof("Found target master alias %v in shard %v/%v", topo.TabletAliasString(si.MasterAlias), keyspace, shard) + wr.Logger().Infof("Found target master alias %v in shard %v/%v", topoproto.TabletAliasString(si.MasterAlias), keyspace, shard) shortCtx, cancel = context.WithTimeout(ctx, *remoteActionsTimeout) ti, err = topo.GetTablet(shortCtx, wr.TopoServer(), si.MasterAlias) cancel() if err != nil { return ti, fmt.Errorf("unable to get master tablet from alias %v in shard %v/%v", - topo.TabletAliasString(si.MasterAlias), keyspace, shard) + topoproto.TabletAliasString(si.MasterAlias), keyspace, shard) } return ti, nil } diff --git a/go/vt/worker/diff_utils.go b/go/vt/worker/diff_utils.go index 77e3cb455d..0fdd9d76e3 100644 --- a/go/vt/worker/diff_utils.go +++ b/go/vt/worker/diff_utils.go @@ -20,6 +20,7 @@ import ( myproto "github.com/youtube/vitess/go/vt/mysqlctl/proto" "github.com/youtube/vitess/go/vt/tabletserver/tabletconn" "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" pb "github.com/youtube/vitess/go/vt/proto/topodata" ) @@ -103,7 +104,7 @@ func uint64FromKeyspaceID(keyspaceID []byte) string { // with the Primary Key columns in front. func TableScan(ctx context.Context, log logutil.Logger, ts topo.Server, tabletAlias *pb.TabletAlias, tableDefinition *myproto.TableDefinition) (*QueryResultReader, error) { sql := fmt.Sprintf("SELECT %v FROM %v ORDER BY %v", strings.Join(orderedColumns(tableDefinition), ", "), tableDefinition.Name, strings.Join(tableDefinition.PrimaryKeyColumns, ", ")) - log.Infof("SQL query for %v/%v: %v", topo.TabletAliasString(tabletAlias), tableDefinition.Name, sql) + log.Infof("SQL query for %v/%v: %v", topoproto.TabletAliasString(tabletAlias), tableDefinition.Name, sql) return NewQueryResultReaderForTablet(ctx, ts, tabletAlias, sql) } @@ -151,7 +152,7 @@ func TableScanByKeyRange(ctx context.Context, log logutil.Logger, ts topo.Server } sql := fmt.Sprintf("SELECT %v FROM %v %vORDER BY %v", strings.Join(orderedColumns(tableDefinition), ", "), tableDefinition.Name, where, strings.Join(tableDefinition.PrimaryKeyColumns, ", ")) - log.Infof("SQL query for %v/%v: %v", topo.TabletAliasString(tabletAlias), tableDefinition.Name, sql) + log.Infof("SQL query for %v/%v: %v", topoproto.TabletAliasString(tabletAlias), tableDefinition.Name, sql) return NewQueryResultReaderForTablet(ctx, ts, tabletAlias, sql) } diff --git a/go/vt/worker/split_clone.go b/go/vt/worker/split_clone.go index 4b284e8b6e..6dadd5cb52 100644 --- a/go/vt/worker/split_clone.go +++ b/go/vt/worker/split_clone.go @@ -20,6 +20,7 @@ import ( "github.com/youtube/vitess/go/vt/mysqlctl" myproto "github.com/youtube/vitess/go/vt/mysqlctl/proto" "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "github.com/youtube/vitess/go/vt/topotools" "github.com/youtube/vitess/go/vt/worker/events" "github.com/youtube/vitess/go/vt/wrangler" @@ -116,7 +117,7 @@ func (scw *SplitCloneWorker) setErrorState(err error) { func (scw *SplitCloneWorker) formatSources() string { result := "" for _, alias := range scw.sourceAliases { - result += " " + topo.TabletAliasString(alias) + result += " " + topoproto.TabletAliasString(alias) } return result } @@ -279,7 +280,7 @@ func (scw *SplitCloneWorker) findTargets(ctx context.Context) error { if err != nil { return fmt.Errorf("FindWorkerTablet() failed for %v/%v/%v: %v", scw.cell, si.Keyspace(), si.ShardName(), err) } - scw.wr.Logger().Infof("Using tablet %v as source for %v/%v", topo.TabletAliasString(scw.sourceAliases[i]), si.Keyspace(), si.ShardName()) + scw.wr.Logger().Infof("Using tablet %v as source for %v/%v", topoproto.TabletAliasString(scw.sourceAliases[i]), si.Keyspace(), si.ShardName()) } // get the tablet info for them, and stop their replication @@ -287,20 +288,20 @@ func (scw *SplitCloneWorker) findTargets(ctx context.Context) error { for i, alias := range scw.sourceAliases { scw.sourceTablets[i], err = scw.wr.TopoServer().GetTablet(ctx, alias) if err != nil { - return fmt.Errorf("cannot read tablet %v: %v", topo.TabletAliasString(alias), err) + return fmt.Errorf("cannot read tablet %v: %v", topoproto.TabletAliasString(alias), err) } shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout) err := scw.wr.TabletManagerClient().StopSlave(shortCtx, scw.sourceTablets[i]) cancel() if err != nil { - return fmt.Errorf("cannot stop replication on tablet %v", topo.TabletAliasString(alias)) + return fmt.Errorf("cannot stop replication on tablet %v", topoproto.TabletAliasString(alias)) } wrangler.RecordStartSlaveAction(scw.cleaner, scw.sourceTablets[i]) action, err := wrangler.FindChangeSlaveTypeActionByTarget(scw.cleaner, alias) if err != nil { - return fmt.Errorf("cannot find ChangeSlaveType action for %v: %v", topo.TabletAliasString(alias), err) + return fmt.Errorf("cannot find ChangeSlaveType action for %v: %v", topoproto.TabletAliasString(alias), err) } action.TabletType = pb.TabletType_SPARE } @@ -383,10 +384,10 @@ func (scw *SplitCloneWorker) copy(ctx context.Context) error { sourceSchemaDefinition, err := scw.wr.GetSchema(shortCtx, scw.sourceAliases[0], nil, scw.excludeTables, true) cancel() if err != nil { - return fmt.Errorf("cannot get schema from source %v: %v", topo.TabletAliasString(scw.sourceAliases[0]), err) + return fmt.Errorf("cannot get schema from source %v: %v", topoproto.TabletAliasString(scw.sourceAliases[0]), err) } if len(sourceSchemaDefinition.TableDefinitions) == 0 { - return fmt.Errorf("no tables matching the table filter in tablet %v", topo.TabletAliasString(scw.sourceAliases[0])) + return fmt.Errorf("no tables matching the table filter in tablet %v", topoproto.TabletAliasString(scw.sourceAliases[0])) } scw.wr.Logger().Infof("Source tablet 0 has %v tables to copy", len(sourceSchemaDefinition.TableDefinitions)) scw.Mu.Lock() diff --git a/go/vt/worker/split_diff.go b/go/vt/worker/split_diff.go index cbe754ea5e..71b7d2365e 100644 --- a/go/vt/worker/split_diff.go +++ b/go/vt/worker/split_diff.go @@ -17,6 +17,7 @@ import ( "github.com/youtube/vitess/go/vt/key" myproto "github.com/youtube/vitess/go/vt/mysqlctl/proto" "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "github.com/youtube/vitess/go/vt/wrangler" pb "github.com/youtube/vitess/go/vt/proto/topodata" @@ -166,7 +167,7 @@ func (sdw *SplitDiffWorker) init(ctx context.Context) error { if len(sdw.shardInfo.SourceShards) == 0 { return fmt.Errorf("shard %v/%v has no source shard", sdw.keyspace, sdw.shard) } - if topo.TabletAliasIsZero(sdw.shardInfo.MasterAlias) { + if !sdw.shardInfo.HasMaster() { return fmt.Errorf("shard %v/%v has no master", sdw.keyspace, sdw.shard) } @@ -308,8 +309,8 @@ func (sdw *SplitDiffWorker) synchronizeReplication(ctx context.Context) error { sdw.wr.Logger().Infof("Restarting filtered replication on master %v", sdw.shardInfo.MasterAlias) shortCtx, cancel = context.WithTimeout(ctx, *remoteActionsTimeout) err = sdw.wr.TabletManagerClient().StartBlp(shortCtx, masterInfo) - if err := sdw.cleaner.RemoveActionByName(wrangler.StartBlpActionName, topo.TabletAliasString(sdw.shardInfo.MasterAlias)); err != nil { - sdw.wr.Logger().Warningf("Cannot find cleaning action %v/%v: %v", wrangler.StartBlpActionName, topo.TabletAliasString(sdw.shardInfo.MasterAlias), err) + if err := sdw.cleaner.RemoveActionByName(wrangler.StartBlpActionName, topoproto.TabletAliasString(sdw.shardInfo.MasterAlias)); err != nil { + sdw.wr.Logger().Warningf("Cannot find cleaning action %v/%v: %v", wrangler.StartBlpActionName, topoproto.TabletAliasString(sdw.shardInfo.MasterAlias), err) } cancel() if err != nil { diff --git a/go/vt/worker/sqldiffer.go b/go/vt/worker/sqldiffer.go index 1e421d675f..c63f780e36 100644 --- a/go/vt/worker/sqldiffer.go +++ b/go/vt/worker/sqldiffer.go @@ -11,7 +11,7 @@ import ( "golang.org/x/net/context" - "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "github.com/youtube/vitess/go/vt/wrangler" pb "github.com/youtube/vitess/go/vt/proto/topodata" @@ -171,7 +171,7 @@ func (worker *SQLDiffWorker) synchronizeReplication(ctx context.Context) error { worker.SetState(WorkerStateSyncReplication) // stop replication on subset slave - worker.wr.Logger().Infof("Stopping replication on subset slave %v", topo.TabletAliasString(worker.subset.alias)) + worker.wr.Logger().Infof("Stopping replication on subset slave %v", topoproto.TabletAliasString(worker.subset.alias)) subsetTablet, err := worker.wr.TopoServer().GetTablet(ctx, worker.subset.alias) if err != nil { return err @@ -180,7 +180,7 @@ func (worker *SQLDiffWorker) synchronizeReplication(ctx context.Context) error { err = worker.wr.TabletManagerClient().StopSlave(shortCtx, subsetTablet) cancel() if err != nil { - return fmt.Errorf("Cannot stop slave %v: %v", topo.TabletAliasString(worker.subset.alias), err) + return fmt.Errorf("Cannot stop slave %v: %v", topoproto.TabletAliasString(worker.subset.alias), err) } if err := checkDone(ctx); err != nil { return err @@ -191,7 +191,7 @@ func (worker *SQLDiffWorker) synchronizeReplication(ctx context.Context) error { wrangler.RecordStartSlaveAction(worker.cleaner, subsetTablet) action, err := wrangler.FindChangeSlaveTypeActionByTarget(worker.cleaner, worker.subset.alias) if err != nil { - return fmt.Errorf("cannot find ChangeSlaveType action for %v: %v", topo.TabletAliasString(worker.subset.alias), err) + return fmt.Errorf("cannot find ChangeSlaveType action for %v: %v", topoproto.TabletAliasString(worker.subset.alias), err) } action.TabletType = pb.TabletType_SPARE @@ -202,7 +202,7 @@ func (worker *SQLDiffWorker) synchronizeReplication(ctx context.Context) error { } // stop replication on superset slave - worker.wr.Logger().Infof("Stopping replication on superset slave %v", topo.TabletAliasString(worker.superset.alias)) + worker.wr.Logger().Infof("Stopping replication on superset slave %v", topoproto.TabletAliasString(worker.superset.alias)) supersetTablet, err := worker.wr.TopoServer().GetTablet(ctx, worker.superset.alias) if err != nil { return err @@ -211,7 +211,7 @@ func (worker *SQLDiffWorker) synchronizeReplication(ctx context.Context) error { err = worker.wr.TabletManagerClient().StopSlave(shortCtx, supersetTablet) cancel() if err != nil { - return fmt.Errorf("Cannot stop slave %v: %v", topo.TabletAliasString(worker.superset.alias), err) + return fmt.Errorf("Cannot stop slave %v: %v", topoproto.TabletAliasString(worker.superset.alias), err) } // change the cleaner actions from ChangeSlaveType(rdonly) @@ -219,7 +219,7 @@ func (worker *SQLDiffWorker) synchronizeReplication(ctx context.Context) error { wrangler.RecordStartSlaveAction(worker.cleaner, supersetTablet) action, err = wrangler.FindChangeSlaveTypeActionByTarget(worker.cleaner, worker.superset.alias) if err != nil { - return fmt.Errorf("cannot find ChangeSlaveType action for %v: %v", topo.TabletAliasString(worker.superset.alias), err) + return fmt.Errorf("cannot find ChangeSlaveType action for %v: %v", topoproto.TabletAliasString(worker.superset.alias), err) } action.TabletType = pb.TabletType_SPARE diff --git a/go/vt/worker/topo_utils.go b/go/vt/worker/topo_utils.go index b30823649a..4791057db4 100644 --- a/go/vt/worker/topo_utils.go +++ b/go/vt/worker/topo_utils.go @@ -12,6 +12,7 @@ import ( "github.com/youtube/vitess/go/vt/servenv" "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "github.com/youtube/vitess/go/vt/wrangler" "golang.org/x/net/context" @@ -97,7 +98,7 @@ func FindWorkerTablet(ctx context.Context, wr *wrangler.Wrangler, cleaner *wrang // We add the tag before calling ChangeSlaveType, so the destination // vttablet reloads the worker URL when it reloads the tablet. ourURL := servenv.ListeningURL.String() - wr.Logger().Infof("Adding tag[worker]=%v to tablet %v", ourURL, topo.TabletAliasString(tabletAlias)) + wr.Logger().Infof("Adding tag[worker]=%v to tablet %v", ourURL, topoproto.TabletAliasString(tabletAlias)) if err := wr.TopoServer().UpdateTabletFields(ctx, tabletAlias, func(tablet *pb.Tablet) error { if tablet.Tags == nil { tablet.Tags = make(map[string]string) @@ -112,7 +113,7 @@ func FindWorkerTablet(ctx context.Context, wr *wrangler.Wrangler, cleaner *wrang // type change in the cleaner. defer wrangler.RecordTabletTagAction(cleaner, tabletAlias, "worker", "") - wr.Logger().Infof("Changing tablet %v to '%v'", topo.TabletAliasString(tabletAlias), pb.TabletType_WORKER) + wr.Logger().Infof("Changing tablet %v to '%v'", topoproto.TabletAliasString(tabletAlias), pb.TabletType_WORKER) shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout) err = wr.ChangeType(shortCtx, tabletAlias, pb.TabletType_WORKER, false /*force*/) cancel() diff --git a/go/vt/worker/vertical_split_clone.go b/go/vt/worker/vertical_split_clone.go index 3b7773e165..460a912848 100644 --- a/go/vt/worker/vertical_split_clone.go +++ b/go/vt/worker/vertical_split_clone.go @@ -20,6 +20,7 @@ import ( "github.com/youtube/vitess/go/vt/mysqlctl" myproto "github.com/youtube/vitess/go/vt/mysqlctl/proto" "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "github.com/youtube/vitess/go/vt/worker/events" "github.com/youtube/vitess/go/vt/wrangler" @@ -117,7 +118,7 @@ func (vscw *VerticalSplitCloneWorker) StatusAsHTML() template.HTML { switch vscw.State { case WorkerStateCopy: result += "Running:
\n" - result += "Copying from: " + topo.TabletAliasString(vscw.sourceAlias) + "
\n" + result += "Copying from: " + topoproto.TabletAliasString(vscw.sourceAlias) + "
\n" statuses, eta := formatTableStatuses(vscw.tableStatus, vscw.startTime) result += "ETA: " + eta.String() + "
\n" result += strings.Join(statuses, "
\n") @@ -139,7 +140,7 @@ func (vscw *VerticalSplitCloneWorker) StatusAsText() string { switch vscw.State { case WorkerStateCopy: result += "Running:\n" - result += "Copying from: " + topo.TabletAliasString(vscw.sourceAlias) + "\n" + result += "Copying from: " + topoproto.TabletAliasString(vscw.sourceAlias) + "\n" statuses, eta := formatTableStatuses(vscw.tableStatus, vscw.startTime) result += "ETA: " + eta.String() + "\n" result += strings.Join(statuses, "\n") @@ -249,12 +250,12 @@ func (vscw *VerticalSplitCloneWorker) findTargets(ctx context.Context) error { if err != nil { return fmt.Errorf("FindWorkerTablet() failed for %v/%v/0: %v", vscw.cell, vscw.sourceKeyspace, err) } - vscw.wr.Logger().Infof("Using tablet %v as the source", topo.TabletAliasString(vscw.sourceAlias)) + vscw.wr.Logger().Infof("Using tablet %v as the source", topoproto.TabletAliasString(vscw.sourceAlias)) // get the tablet info for it vscw.sourceTablet, err = vscw.wr.TopoServer().GetTablet(ctx, vscw.sourceAlias) if err != nil { - return fmt.Errorf("cannot read tablet %v: %v", topo.TabletAliasString(vscw.sourceAlias), err) + return fmt.Errorf("cannot read tablet %v: %v", topoproto.TabletAliasString(vscw.sourceAlias), err) } // stop replication on it @@ -262,13 +263,13 @@ func (vscw *VerticalSplitCloneWorker) findTargets(ctx context.Context) error { err = vscw.wr.TabletManagerClient().StopSlave(shortCtx, vscw.sourceTablet) cancel() if err != nil { - return fmt.Errorf("cannot stop replication on tablet %v", topo.TabletAliasString(vscw.sourceAlias)) + return fmt.Errorf("cannot stop replication on tablet %v", topoproto.TabletAliasString(vscw.sourceAlias)) } wrangler.RecordStartSlaveAction(vscw.cleaner, vscw.sourceTablet) action, err := wrangler.FindChangeSlaveTypeActionByTarget(vscw.cleaner, vscw.sourceAlias) if err != nil { - return fmt.Errorf("cannot find ChangeSlaveType action for %v: %v", topo.TabletAliasString(vscw.sourceAlias), err) + return fmt.Errorf("cannot find ChangeSlaveType action for %v: %v", topoproto.TabletAliasString(vscw.sourceAlias), err) } action.TabletType = pb.TabletType_SPARE @@ -336,7 +337,7 @@ func (vscw *VerticalSplitCloneWorker) copy(ctx context.Context) error { sourceSchemaDefinition, err := vscw.wr.GetSchema(shortCtx, vscw.sourceAlias, vscw.tables, nil, true) cancel() if err != nil { - return fmt.Errorf("cannot get schema from source %v: %v", topo.TabletAliasString(vscw.sourceAlias), err) + return fmt.Errorf("cannot get schema from source %v: %v", topoproto.TabletAliasString(vscw.sourceAlias), err) } if len(sourceSchemaDefinition.TableDefinitions) == 0 { return fmt.Errorf("no tables matching the table filter") @@ -429,7 +430,7 @@ func (vscw *VerticalSplitCloneWorker) copy(ctx context.Context) error { vscw.tableStatus[tableIndex].threadStarted() // build the query, and start the streaming - selectSQL := buildSQLFromChunks(vscw.wr, td, chunks, chunkIndex, topo.TabletAliasString(vscw.sourceAlias)) + selectSQL := buildSQLFromChunks(vscw.wr, td, chunks, chunkIndex, topoproto.TabletAliasString(vscw.sourceAlias)) qrr, err := NewQueryResultReaderForTablet(ctx, vscw.wr.TopoServer(), vscw.sourceAlias, selectSQL) if err != nil { processError("NewQueryResultReaderForTablet failed: %v", err) diff --git a/go/vt/worker/vertical_split_diff.go b/go/vt/worker/vertical_split_diff.go index b273edd3fd..c8b078b114 100644 --- a/go/vt/worker/vertical_split_diff.go +++ b/go/vt/worker/vertical_split_diff.go @@ -17,6 +17,7 @@ import ( "github.com/youtube/vitess/go/vt/concurrency" myproto "github.com/youtube/vitess/go/vt/mysqlctl/proto" "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "github.com/youtube/vitess/go/vt/wrangler" pb "github.com/youtube/vitess/go/vt/proto/topodata" @@ -175,7 +176,7 @@ func (vsdw *VerticalSplitDiffWorker) init(ctx context.Context) error { if len(vsdw.shardInfo.SourceShards[0].Tables) == 0 { return fmt.Errorf("shard %v/%v has no tables in source shard[0]", vsdw.keyspace, vsdw.shard) } - if topo.TabletAliasIsZero(vsdw.shardInfo.MasterAlias) { + if !vsdw.shardInfo.HasMaster() { return fmt.Errorf("shard %v/%v has no master", vsdw.keyspace, vsdw.shard) } @@ -228,16 +229,16 @@ func (vsdw *VerticalSplitDiffWorker) synchronizeReplication(ctx context.Context) masterInfo, err := vsdw.wr.TopoServer().GetTablet(ctx, vsdw.shardInfo.MasterAlias) if err != nil { - return fmt.Errorf("synchronizeReplication: cannot get Tablet record for master %v: %v", topo.TabletAliasString(vsdw.shardInfo.MasterAlias), err) + return fmt.Errorf("synchronizeReplication: cannot get Tablet record for master %v: %v", topoproto.TabletAliasString(vsdw.shardInfo.MasterAlias), err) } // 1 - stop the master binlog replication, get its current position - vsdw.wr.Logger().Infof("Stopping master binlog replication on %v", topo.TabletAliasString(vsdw.shardInfo.MasterAlias)) + vsdw.wr.Logger().Infof("Stopping master binlog replication on %v", topoproto.TabletAliasString(vsdw.shardInfo.MasterAlias)) shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout) blpPositionList, err := vsdw.wr.TabletManagerClient().StopBlp(shortCtx, masterInfo) cancel() if err != nil { - return fmt.Errorf("StopBlp on master %v failed: %v", topo.TabletAliasString(vsdw.shardInfo.MasterAlias), err) + return fmt.Errorf("StopBlp on master %v failed: %v", topoproto.TabletAliasString(vsdw.shardInfo.MasterAlias), err) } wrangler.RecordStartBlpAction(vsdw.cleaner, masterInfo) @@ -254,7 +255,7 @@ func (vsdw *VerticalSplitDiffWorker) synchronizeReplication(ctx context.Context) } // stop replication - vsdw.wr.Logger().Infof("Stopping slave %v at a minimum of %v", topo.TabletAliasString(vsdw.sourceAlias), pos.Position) + vsdw.wr.Logger().Infof("Stopping slave %v at a minimum of %v", topoproto.TabletAliasString(vsdw.sourceAlias), pos.Position) sourceTablet, err := vsdw.wr.TopoServer().GetTablet(ctx, vsdw.sourceAlias) if err != nil { return err @@ -263,7 +264,7 @@ func (vsdw *VerticalSplitDiffWorker) synchronizeReplication(ctx context.Context) stoppedAt, err := vsdw.wr.TabletManagerClient().StopSlaveMinimum(shortCtx, sourceTablet, pos.Position, *remoteActionsTimeout) cancel() if err != nil { - return fmt.Errorf("cannot stop slave %v at right binlog position %v: %v", topo.TabletAliasString(vsdw.sourceAlias), pos.Position, err) + return fmt.Errorf("cannot stop slave %v at right binlog position %v: %v", topoproto.TabletAliasString(vsdw.sourceAlias), pos.Position, err) } stopPositionList.Entries[0].Uid = ss.Uid stopPositionList.Entries[0].Position = stoppedAt @@ -273,23 +274,23 @@ func (vsdw *VerticalSplitDiffWorker) synchronizeReplication(ctx context.Context) wrangler.RecordStartSlaveAction(vsdw.cleaner, sourceTablet) action, err := wrangler.FindChangeSlaveTypeActionByTarget(vsdw.cleaner, vsdw.sourceAlias) if err != nil { - return fmt.Errorf("cannot find ChangeSlaveType action for %v: %v", topo.TabletAliasString(vsdw.sourceAlias), err) + return fmt.Errorf("cannot find ChangeSlaveType action for %v: %v", topoproto.TabletAliasString(vsdw.sourceAlias), err) } action.TabletType = pb.TabletType_SPARE // 3 - ask the master of the destination shard to resume filtered // replication up to the new list of positions - vsdw.wr.Logger().Infof("Restarting master %v until it catches up to %v", topo.TabletAliasString(vsdw.shardInfo.MasterAlias), stopPositionList) + vsdw.wr.Logger().Infof("Restarting master %v until it catches up to %v", topoproto.TabletAliasString(vsdw.shardInfo.MasterAlias), stopPositionList) shortCtx, cancel = context.WithTimeout(ctx, *remoteActionsTimeout) masterPos, err := vsdw.wr.TabletManagerClient().RunBlpUntil(shortCtx, masterInfo, &stopPositionList, *remoteActionsTimeout) cancel() if err != nil { - return fmt.Errorf("RunBlpUntil on %v until %v failed: %v", topo.TabletAliasString(vsdw.shardInfo.MasterAlias), stopPositionList, err) + return fmt.Errorf("RunBlpUntil on %v until %v failed: %v", topoproto.TabletAliasString(vsdw.shardInfo.MasterAlias), stopPositionList, err) } // 4 - wait until the destination tablet is equal or passed // that master binlog position, and stop its replication. - vsdw.wr.Logger().Infof("Waiting for destination tablet %v to catch up to %v", topo.TabletAliasString(vsdw.destinationAlias), masterPos) + vsdw.wr.Logger().Infof("Waiting for destination tablet %v to catch up to %v", topoproto.TabletAliasString(vsdw.destinationAlias), masterPos) destinationTablet, err := vsdw.wr.TopoServer().GetTablet(ctx, vsdw.destinationAlias) if err != nil { return err @@ -298,25 +299,25 @@ func (vsdw *VerticalSplitDiffWorker) synchronizeReplication(ctx context.Context) _, err = vsdw.wr.TabletManagerClient().StopSlaveMinimum(shortCtx, destinationTablet, masterPos, *remoteActionsTimeout) cancel() if err != nil { - return fmt.Errorf("StopSlaveMinimum on %v at %v failed: %v", topo.TabletAliasString(vsdw.destinationAlias), masterPos, err) + return fmt.Errorf("StopSlaveMinimum on %v at %v failed: %v", topoproto.TabletAliasString(vsdw.destinationAlias), masterPos, err) } wrangler.RecordStartSlaveAction(vsdw.cleaner, destinationTablet) action, err = wrangler.FindChangeSlaveTypeActionByTarget(vsdw.cleaner, vsdw.destinationAlias) if err != nil { - return fmt.Errorf("cannot find ChangeSlaveType action for %v: %v", topo.TabletAliasString(vsdw.destinationAlias), err) + return fmt.Errorf("cannot find ChangeSlaveType action for %v: %v", topoproto.TabletAliasString(vsdw.destinationAlias), err) } action.TabletType = pb.TabletType_SPARE // 5 - restart filtered replication on destination master - vsdw.wr.Logger().Infof("Restarting filtered replication on master %v", topo.TabletAliasString(vsdw.shardInfo.MasterAlias)) + vsdw.wr.Logger().Infof("Restarting filtered replication on master %v", topoproto.TabletAliasString(vsdw.shardInfo.MasterAlias)) shortCtx, cancel = context.WithTimeout(ctx, *remoteActionsTimeout) err = vsdw.wr.TabletManagerClient().StartBlp(shortCtx, masterInfo) - if err := vsdw.cleaner.RemoveActionByName(wrangler.StartBlpActionName, topo.TabletAliasString(vsdw.shardInfo.MasterAlias)); err != nil { - vsdw.wr.Logger().Warningf("Cannot find cleaning action %v/%v: %v", wrangler.StartBlpActionName, topo.TabletAliasString(vsdw.shardInfo.MasterAlias), err) + if err := vsdw.cleaner.RemoveActionByName(wrangler.StartBlpActionName, topoproto.TabletAliasString(vsdw.shardInfo.MasterAlias)); err != nil { + vsdw.wr.Logger().Warningf("Cannot find cleaning action %v/%v: %v", wrangler.StartBlpActionName, topoproto.TabletAliasString(vsdw.shardInfo.MasterAlias), err) } cancel() if err != nil { - return fmt.Errorf("StartBlp on %v failed: %v", topo.TabletAliasString(vsdw.shardInfo.MasterAlias), err) + return fmt.Errorf("StartBlp on %v failed: %v", topoproto.TabletAliasString(vsdw.shardInfo.MasterAlias), err) } return nil @@ -341,7 +342,7 @@ func (vsdw *VerticalSplitDiffWorker) diff(ctx context.Context) error { shortCtx, vsdw.destinationAlias, nil /* tables */, vsdw.excludeTables, false /* includeViews */) cancel() rec.RecordError(err) - vsdw.wr.Logger().Infof("Got schema from destination %v", topo.TabletAliasString(vsdw.destinationAlias)) + vsdw.wr.Logger().Infof("Got schema from destination %v", topoproto.TabletAliasString(vsdw.destinationAlias)) wg.Done() }() wg.Add(1) @@ -352,7 +353,7 @@ func (vsdw *VerticalSplitDiffWorker) diff(ctx context.Context) error { shortCtx, vsdw.sourceAlias, nil /* tables */, vsdw.excludeTables, false /* includeViews */) cancel() rec.RecordError(err) - vsdw.wr.Logger().Infof("Got schema from source %v", topo.TabletAliasString(vsdw.sourceAlias)) + vsdw.wr.Logger().Infof("Got schema from source %v", topoproto.TabletAliasString(vsdw.sourceAlias)) wg.Done() }() wg.Wait() diff --git a/go/vt/wrangler/cleaner.go b/go/vt/wrangler/cleaner.go index 60d5f21438..7d093591a8 100644 --- a/go/vt/wrangler/cleaner.go +++ b/go/vt/wrangler/cleaner.go @@ -11,6 +11,7 @@ import ( "github.com/youtube/vitess/go/vt/concurrency" "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "golang.org/x/net/context" pb "github.com/youtube/vitess/go/vt/proto/topodata" @@ -142,7 +143,7 @@ const ChangeSlaveTypeActionName = "ChangeSlaveTypeAction" // RecordChangeSlaveTypeAction records a new ChangeSlaveTypeAction // into the specified Cleaner func RecordChangeSlaveTypeAction(cleaner *Cleaner, tabletAlias *pb.TabletAlias, tabletType pb.TabletType) { - cleaner.Record(ChangeSlaveTypeActionName, topo.TabletAliasString(tabletAlias), &ChangeSlaveTypeAction{ + cleaner.Record(ChangeSlaveTypeActionName, topoproto.TabletAliasString(tabletAlias), &ChangeSlaveTypeAction{ TabletAlias: tabletAlias, TabletType: tabletType, }) @@ -150,7 +151,7 @@ func RecordChangeSlaveTypeAction(cleaner *Cleaner, tabletAlias *pb.TabletAlias, // FindChangeSlaveTypeActionByTarget finds the first action for the target func FindChangeSlaveTypeActionByTarget(cleaner *Cleaner, tabletAlias *pb.TabletAlias) (*ChangeSlaveTypeAction, error) { - action, err := cleaner.GetActionByName(ChangeSlaveTypeActionName, topo.TabletAliasString(tabletAlias)) + action, err := cleaner.GetActionByName(ChangeSlaveTypeActionName, topoproto.TabletAliasString(tabletAlias)) if err != nil { return nil, err } @@ -184,7 +185,7 @@ const TabletTagActionName = "TabletTagAction" // RecordTabletTagAction records a new TabletTagAction // into the specified Cleaner func RecordTabletTagAction(cleaner *Cleaner, tabletAlias *pb.TabletAlias, name, value string) { - cleaner.Record(TabletTagActionName, topo.TabletAliasString(tabletAlias), &TabletTagAction{ + cleaner.Record(TabletTagActionName, topoproto.TabletAliasString(tabletAlias), &TabletTagAction{ TabletAlias: tabletAlias, Name: name, Value: value, @@ -221,7 +222,7 @@ const StartSlaveActionName = "StartSlaveAction" // RecordStartSlaveAction records a new StartSlaveAction // into the specified Cleaner func RecordStartSlaveAction(cleaner *Cleaner, tabletInfo *topo.TabletInfo) { - cleaner.Record(StartSlaveActionName, topo.TabletAliasString(tabletInfo.Alias), &StartSlaveAction{ + cleaner.Record(StartSlaveActionName, topoproto.TabletAliasString(tabletInfo.Alias), &StartSlaveAction{ TabletInfo: tabletInfo, }) } @@ -246,7 +247,7 @@ const StartBlpActionName = "StartBlpAction" // RecordStartBlpAction records a new StartBlpAction // into the specified Cleaner func RecordStartBlpAction(cleaner *Cleaner, tabletInfo *topo.TabletInfo) { - cleaner.Record(StartBlpActionName, topo.TabletAliasString(tabletInfo.Alias), &StartBlpAction{ + cleaner.Record(StartBlpActionName, topoproto.TabletAliasString(tabletInfo.Alias), &StartBlpAction{ TabletInfo: tabletInfo, }) } diff --git a/go/vt/wrangler/keyspace.go b/go/vt/wrangler/keyspace.go index ad4717c33f..188feef097 100644 --- a/go/vt/wrangler/keyspace.go +++ b/go/vt/wrangler/keyspace.go @@ -15,6 +15,7 @@ import ( myproto "github.com/youtube/vitess/go/vt/mysqlctl/proto" "github.com/youtube/vitess/go/vt/tabletmanager/actionnode" "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "github.com/youtube/vitess/go/vt/topotools" "github.com/youtube/vitess/go/vt/topotools/events" "golang.org/x/net/context" @@ -203,7 +204,7 @@ func (wr *Wrangler) getMastersPosition(ctx context.Context, shards []*topo.Shard wg.Add(1) go func(si *topo.ShardInfo) { defer wg.Done() - wr.Logger().Infof("Gathering master position for %v", topo.TabletAliasString(si.MasterAlias)) + wr.Logger().Infof("Gathering master position for %v", topoproto.TabletAliasString(si.MasterAlias)) ti, err := wr.ts.GetTablet(ctx, si.MasterAlias) if err != nil { rec.RecordError(err) @@ -216,7 +217,7 @@ func (wr *Wrangler) getMastersPosition(ctx context.Context, shards []*topo.Shard return } - wr.Logger().Infof("Got master position for %v", topo.TabletAliasString(si.MasterAlias)) + wr.Logger().Infof("Got master position for %v", topoproto.TabletAliasString(si.MasterAlias)) mu.Lock() result[si] = pos mu.Unlock() @@ -247,7 +248,7 @@ func (wr *Wrangler) waitForFilteredReplication(ctx context.Context, sourcePositi } // and wait for it - wr.Logger().Infof("Waiting for %v to catch up", topo.TabletAliasString(si.MasterAlias)) + wr.Logger().Infof("Waiting for %v to catch up", topoproto.TabletAliasString(si.MasterAlias)) tablet, err := wr.ts.GetTablet(ctx, si.MasterAlias) if err != nil { rec.RecordError(err) @@ -257,7 +258,7 @@ func (wr *Wrangler) waitForFilteredReplication(ctx context.Context, sourcePositi if err := wr.tmc.WaitBlpPosition(ctx, tablet, blpPosition, waitTime); err != nil { rec.RecordError(err) } else { - wr.Logger().Infof("%v caught up", topo.TabletAliasString(si.MasterAlias)) + wr.Logger().Infof("%v caught up", topoproto.TabletAliasString(si.MasterAlias)) } } }(si) @@ -274,7 +275,7 @@ func (wr *Wrangler) refreshMasters(ctx context.Context, shards []*topo.ShardInfo wg.Add(1) go func(si *topo.ShardInfo) { defer wg.Done() - wr.Logger().Infof("RefreshState master %v", topo.TabletAliasString(si.MasterAlias)) + wr.Logger().Infof("RefreshState master %v", topoproto.TabletAliasString(si.MasterAlias)) ti, err := wr.ts.GetTablet(ctx, si.MasterAlias) if err != nil { rec.RecordError(err) @@ -284,7 +285,7 @@ func (wr *Wrangler) refreshMasters(ctx context.Context, shards []*topo.ShardInfo if err := wr.tmc.RefreshState(ctx, ti); err != nil { rec.RecordError(err) } else { - wr.Logger().Infof("%v responded", topo.TabletAliasString(si.MasterAlias)) + wr.Logger().Infof("%v responded", topoproto.TabletAliasString(si.MasterAlias)) } }(si) } diff --git a/go/vt/wrangler/permissions.go b/go/vt/wrangler/permissions.go index 6c685b54fb..7d86d7f605 100644 --- a/go/vt/wrangler/permissions.go +++ b/go/vt/wrangler/permissions.go @@ -13,6 +13,7 @@ import ( "github.com/youtube/vitess/go/vt/concurrency" myproto "github.com/youtube/vitess/go/vt/mysqlctl/proto" "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "golang.org/x/net/context" pb "github.com/youtube/vitess/go/vt/proto/topodata" @@ -31,15 +32,15 @@ func (wr *Wrangler) GetPermissions(ctx context.Context, tabletAlias *pb.TabletAl // diffPermissions is a helper method to asynchronously diff a permissions func (wr *Wrangler) diffPermissions(ctx context.Context, masterPermissions *myproto.Permissions, masterAlias *pb.TabletAlias, alias *pb.TabletAlias, wg *sync.WaitGroup, er concurrency.ErrorRecorder) { defer wg.Done() - log.Infof("Gathering permissions for %v", topo.TabletAliasString(alias)) + log.Infof("Gathering permissions for %v", topoproto.TabletAliasString(alias)) slavePermissions, err := wr.GetPermissions(ctx, alias) if err != nil { er.RecordError(err) return } - log.Infof("Diffing permissions for %v", topo.TabletAliasString(alias)) - myproto.DiffPermissions(topo.TabletAliasString(masterAlias), masterPermissions, topo.TabletAliasString(alias), slavePermissions, er) + log.Infof("Diffing permissions for %v", topoproto.TabletAliasString(alias)) + myproto.DiffPermissions(topoproto.TabletAliasString(masterAlias), masterPermissions, topoproto.TabletAliasString(alias), slavePermissions, er) } // ValidatePermissionsShard validates all the permissions are the same @@ -51,10 +52,10 @@ func (wr *Wrangler) ValidatePermissionsShard(ctx context.Context, keyspace, shar } // get permissions from the master, or error - if topo.TabletAliasIsZero(si.MasterAlias) { + if !si.HasMaster() { return fmt.Errorf("No master in shard %v/%v", keyspace, shard) } - log.Infof("Gathering permissions for master %v", topo.TabletAliasString(si.MasterAlias)) + log.Infof("Gathering permissions for master %v", topoproto.TabletAliasString(si.MasterAlias)) masterPermissions, err := wr.GetPermissions(ctx, si.MasterAlias) if err != nil { return err @@ -71,7 +72,7 @@ func (wr *Wrangler) ValidatePermissionsShard(ctx context.Context, keyspace, shar er := concurrency.AllErrorRecorder{} wg := sync.WaitGroup{} for _, alias := range aliases { - if topo.TabletAliasEqual(alias, si.MasterAlias) { + if topoproto.TabletAliasEqual(alias, si.MasterAlias) { continue } wg.Add(1) @@ -107,11 +108,11 @@ func (wr *Wrangler) ValidatePermissionsKeyspace(ctx context.Context, keyspace st if err != nil { return err } - if topo.TabletAliasIsZero(si.MasterAlias) { + if !si.HasMaster() { return fmt.Errorf("No master in shard %v/%v", keyspace, shards[0]) } referenceAlias := si.MasterAlias - log.Infof("Gathering permissions for reference master %v", topo.TabletAliasString(referenceAlias)) + log.Infof("Gathering permissions for reference master %v", topoproto.TabletAliasString(referenceAlias)) referencePermissions, err := wr.GetPermissions(ctx, si.MasterAlias) if err != nil { return err @@ -128,7 +129,7 @@ func (wr *Wrangler) ValidatePermissionsKeyspace(ctx context.Context, keyspace st } for _, alias := range aliases { - if topo.TabletAliasEqual(alias, si.MasterAlias) { + if topoproto.TabletAliasEqual(alias, si.MasterAlias) { continue } diff --git a/go/vt/wrangler/reparent.go b/go/vt/wrangler/reparent.go index 478093e486..bb9779a8e5 100644 --- a/go/vt/wrangler/reparent.go +++ b/go/vt/wrangler/reparent.go @@ -18,6 +18,7 @@ import ( myproto "github.com/youtube/vitess/go/vt/mysqlctl/proto" "github.com/youtube/vitess/go/vt/tabletmanager/actionnode" "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "github.com/youtube/vitess/go/vt/topotools" "github.com/youtube/vitess/go/vt/topotools/events" "golang.org/x/net/context" @@ -126,7 +127,7 @@ func (wr *Wrangler) ReparentTablet(ctx context.Context, tabletAlias *pb.TabletAl if err != nil { return err } - if topo.TabletAliasIsZero(shardInfo.MasterAlias) { + if !shardInfo.HasMaster() { return fmt.Errorf("no master tablet for shard %v/%v", ti.Keyspace, ti.Shard) } @@ -137,10 +138,10 @@ func (wr *Wrangler) ReparentTablet(ctx context.Context, tabletAlias *pb.TabletAl // Basic sanity checking. if masterTi.Type != pb.TabletType_MASTER { - return fmt.Errorf("TopologyServer has inconsistent state for shard master %v", topo.TabletAliasString(shardInfo.MasterAlias)) + return fmt.Errorf("TopologyServer has inconsistent state for shard master %v", topoproto.TabletAliasString(shardInfo.MasterAlias)) } if masterTi.Keyspace != ti.Keyspace || masterTi.Shard != ti.Shard { - return fmt.Errorf("master %v and potential slave not in same keyspace/shard", topo.TabletAliasString(shardInfo.MasterAlias)) + return fmt.Errorf("master %v and potential slave not in same keyspace/shard", topoproto.TabletAliasString(shardInfo.MasterAlias)) } // and do the remote command @@ -187,35 +188,35 @@ func (wr *Wrangler) initShardMasterLocked(ctx context.Context, ev *events.Repare // Check the master elect is in tabletMap masterElectTabletInfo, ok := tabletMap[*masterElectTabletAlias] if !ok { - return fmt.Errorf("master-elect tablet %v is not in the shard", topo.TabletAliasString(masterElectTabletAlias)) + return fmt.Errorf("master-elect tablet %v is not in the shard", topoproto.TabletAliasString(masterElectTabletAlias)) } ev.NewMaster = *masterElectTabletInfo.Tablet // Check the master is the only master is the shard, or -force was used. _, masterTabletMap := topotools.SortedTabletMap(tabletMap) - if !topo.TabletAliasEqual(shardInfo.MasterAlias, masterElectTabletAlias) { + if !topoproto.TabletAliasEqual(shardInfo.MasterAlias, masterElectTabletAlias) { if !force { - return fmt.Errorf("master-elect tablet %v is not the shard master, use -force to proceed anyway", topo.TabletAliasString(masterElectTabletAlias)) + return fmt.Errorf("master-elect tablet %v is not the shard master, use -force to proceed anyway", topoproto.TabletAliasString(masterElectTabletAlias)) } - wr.logger.Warningf("master-elect tablet %v is not the shard master, proceeding anyway as -force was used", topo.TabletAliasString(masterElectTabletAlias)) + wr.logger.Warningf("master-elect tablet %v is not the shard master, proceeding anyway as -force was used", topoproto.TabletAliasString(masterElectTabletAlias)) } if _, ok := masterTabletMap[*masterElectTabletAlias]; !ok { if !force { - return fmt.Errorf("master-elect tablet %v is not a master in the shard, use -force to proceed anyway", topo.TabletAliasString(masterElectTabletAlias)) + return fmt.Errorf("master-elect tablet %v is not a master in the shard, use -force to proceed anyway", topoproto.TabletAliasString(masterElectTabletAlias)) } - wr.logger.Warningf("master-elect tablet %v is not a master in the shard, proceeding anyway as -force was used", topo.TabletAliasString(masterElectTabletAlias)) + wr.logger.Warningf("master-elect tablet %v is not a master in the shard, proceeding anyway as -force was used", topoproto.TabletAliasString(masterElectTabletAlias)) } haveOtherMaster := false for alias, ti := range masterTabletMap { - if !topo.TabletAliasEqual(&alias, masterElectTabletAlias) && ti.Type != pb.TabletType_SCRAP { + if !topoproto.TabletAliasEqual(&alias, masterElectTabletAlias) && ti.Type != pb.TabletType_SCRAP { haveOtherMaster = true } } if haveOtherMaster { if !force { - return fmt.Errorf("master-elect tablet %v is not the only master in the shard, use -force to proceed anyway", topo.TabletAliasString(masterElectTabletAlias)) + return fmt.Errorf("master-elect tablet %v is not the only master in the shard, use -force to proceed anyway", topoproto.TabletAliasString(masterElectTabletAlias)) } - wr.logger.Warningf("master-elect tablet %v is not the only master in the shard, proceeding anyway as -force was used", topo.TabletAliasString(masterElectTabletAlias)) + wr.logger.Warningf("master-elect tablet %v is not the only master in the shard, proceeding anyway as -force was used", topoproto.TabletAliasString(masterElectTabletAlias)) } // First phase: reset replication on all tablets. If anyone fails, @@ -229,9 +230,9 @@ func (wr *Wrangler) initShardMasterLocked(ctx context.Context, ev *events.Repare wg.Add(1) go func(alias pb.TabletAlias, tabletInfo *topo.TabletInfo) { defer wg.Done() - wr.logger.Infof("resetting replication on tablet %v", topo.TabletAliasString(&alias)) + wr.logger.Infof("resetting replication on tablet %v", topoproto.TabletAliasString(&alias)) if err := wr.TabletManagerClient().ResetReplication(ctx, tabletInfo); err != nil { - rec.RecordError(fmt.Errorf("Tablet %v ResetReplication failed (either fix it, or Scrap it): %v", topo.TabletAliasString(&alias), err)) + rec.RecordError(fmt.Errorf("Tablet %v ResetReplication failed (either fix it, or Scrap it): %v", topoproto.TabletAliasString(&alias), err)) } }(alias, tabletInfo) } @@ -242,7 +243,7 @@ func (wr *Wrangler) initShardMasterLocked(ctx context.Context, ev *events.Repare // Tell the new master to break its slaves, return its replication // position - wr.logger.Infof("initializing master on %v", topo.TabletAliasString(masterElectTabletAlias)) + wr.logger.Infof("initializing master on %v", topoproto.TabletAliasString(masterElectTabletAlias)) event.DispatchUpdate(ev, "initializing master") rp, err := wr.TabletManagerClient().InitMaster(ctx, masterElectTabletInfo) if err != nil { @@ -261,20 +262,20 @@ func (wr *Wrangler) initShardMasterLocked(ctx context.Context, ev *events.Repare wgSlaves := sync.WaitGroup{} var masterErr error for alias, tabletInfo := range tabletMap { - if topo.TabletAliasEqual(&alias, masterElectTabletAlias) { + if topoproto.TabletAliasEqual(&alias, masterElectTabletAlias) { wgMaster.Add(1) go func(alias pb.TabletAlias, tabletInfo *topo.TabletInfo) { defer wgMaster.Done() - wr.logger.Infof("populating reparent journal on new master %v", topo.TabletAliasString(&alias)) + wr.logger.Infof("populating reparent journal on new master %v", topoproto.TabletAliasString(&alias)) masterErr = wr.TabletManagerClient().PopulateReparentJournal(ctx, tabletInfo, now, initShardMasterOperation, &alias, rp) }(alias, tabletInfo) } else { wgSlaves.Add(1) go func(alias pb.TabletAlias, tabletInfo *topo.TabletInfo) { defer wgSlaves.Done() - wr.logger.Infof("initializing slave %v", topo.TabletAliasString(&alias)) + wr.logger.Infof("initializing slave %v", topoproto.TabletAliasString(&alias)) if err := wr.TabletManagerClient().InitSlave(ctx, tabletInfo, masterElectTabletAlias, rp, now); err != nil { - rec.RecordError(fmt.Errorf("Tablet %v InitSlave failed: %v", topo.TabletAliasString(&alias), err)) + rec.RecordError(fmt.Errorf("Tablet %v InitSlave failed: %v", topoproto.TabletAliasString(&alias), err)) } }(alias, tabletInfo) } @@ -287,7 +288,7 @@ func (wr *Wrangler) initShardMasterLocked(ctx context.Context, ev *events.Repare wgSlaves.Wait() return fmt.Errorf("failed to PopulateReparentJournal on master: %v", masterErr) } - if !topo.TabletAliasEqual(shardInfo.MasterAlias, masterElectTabletAlias) { + if !topoproto.TabletAliasEqual(shardInfo.MasterAlias, masterElectTabletAlias) { shardInfo.MasterAlias = masterElectTabletAlias if err := topo.UpdateShard(ctx, wr.ts, shardInfo); err != nil { wgSlaves.Wait() @@ -352,15 +353,15 @@ func (wr *Wrangler) plannedReparentShardLocked(ctx context.Context, ev *events.R // Check corner cases we're going to depend on masterElectTabletInfo, ok := tabletMap[*masterElectTabletAlias] if !ok { - return fmt.Errorf("master-elect tablet %v is not in the shard", topo.TabletAliasString(masterElectTabletAlias)) + return fmt.Errorf("master-elect tablet %v is not in the shard", topoproto.TabletAliasString(masterElectTabletAlias)) } ev.NewMaster = *masterElectTabletInfo.Tablet - if topo.TabletAliasEqual(shardInfo.MasterAlias, masterElectTabletAlias) { - return fmt.Errorf("master-elect tablet %v is already the master", topo.TabletAliasString(masterElectTabletAlias)) + if topoproto.TabletAliasEqual(shardInfo.MasterAlias, masterElectTabletAlias) { + return fmt.Errorf("master-elect tablet %v is already the master", topoproto.TabletAliasString(masterElectTabletAlias)) } oldMasterTabletInfo, ok := tabletMap[*shardInfo.MasterAlias] if !ok { - return fmt.Errorf("old master tablet %v is not in the shard", topo.TabletAliasString(shardInfo.MasterAlias)) + return fmt.Errorf("old master tablet %v is not in the shard", topoproto.TabletAliasString(shardInfo.MasterAlias)) } ev.OldMaster = *oldMasterTabletInfo.Tablet @@ -369,16 +370,16 @@ func (wr *Wrangler) plannedReparentShardLocked(ctx context.Context, ev *events.R event.DispatchUpdate(ev, "demoting old master") rp, err := wr.tmc.DemoteMaster(ctx, oldMasterTabletInfo) if err != nil { - return fmt.Errorf("old master tablet %v DemoteMaster failed: %v", topo.TabletAliasString(shardInfo.MasterAlias), err) + return fmt.Errorf("old master tablet %v DemoteMaster failed: %v", topoproto.TabletAliasString(shardInfo.MasterAlias), err) } // Wait on the master-elect tablet until it reaches that position, // then promote it - wr.logger.Infof("promote slave %v", topo.TabletAliasString(masterElectTabletAlias)) + wr.logger.Infof("promote slave %v", topoproto.TabletAliasString(masterElectTabletAlias)) event.DispatchUpdate(ev, "promoting slave") rp, err = wr.tmc.PromoteSlaveWhenCaughtUp(ctx, masterElectTabletInfo, rp) if err != nil { - return fmt.Errorf("master-elect tablet %v failed to catch up with replication or be upgraded to master: %v", topo.TabletAliasString(masterElectTabletAlias), err) + return fmt.Errorf("master-elect tablet %v failed to catch up with replication or be upgraded to master: %v", topoproto.TabletAliasString(masterElectTabletAlias), err) } // Go through all the tablets: @@ -391,22 +392,22 @@ func (wr *Wrangler) plannedReparentShardLocked(ctx context.Context, ev *events.R rec := concurrency.AllErrorRecorder{} var masterErr error for alias, tabletInfo := range tabletMap { - if topo.TabletAliasEqual(&alias, masterElectTabletAlias) { + if topoproto.TabletAliasEqual(&alias, masterElectTabletAlias) { wgMaster.Add(1) go func(alias pb.TabletAlias, tabletInfo *topo.TabletInfo) { defer wgMaster.Done() - wr.logger.Infof("populating reparent journal on new master %v", topo.TabletAliasString(&alias)) + wr.logger.Infof("populating reparent journal on new master %v", topoproto.TabletAliasString(&alias)) masterErr = wr.TabletManagerClient().PopulateReparentJournal(ctx, tabletInfo, now, plannedReparentShardOperation, &alias, rp) }(alias, tabletInfo) } else { wgSlaves.Add(1) go func(alias pb.TabletAlias, tabletInfo *topo.TabletInfo) { defer wgSlaves.Done() - wr.logger.Infof("setting new master on slave %v", topo.TabletAliasString(&alias)) + wr.logger.Infof("setting new master on slave %v", topoproto.TabletAliasString(&alias)) // also restart replication on old master - forceStartSlave := topo.TabletAliasEqual(&alias, oldMasterTabletInfo.Alias) + forceStartSlave := topoproto.TabletAliasEqual(&alias, oldMasterTabletInfo.Alias) if err := wr.TabletManagerClient().SetMaster(ctx, tabletInfo, masterElectTabletAlias, now, forceStartSlave); err != nil { - rec.RecordError(fmt.Errorf("Tablet %v SetMaster failed: %v", topo.TabletAliasString(&alias), err)) + rec.RecordError(fmt.Errorf("Tablet %v SetMaster failed: %v", topoproto.TabletAliasString(&alias), err)) return } }(alias, tabletInfo) @@ -484,16 +485,16 @@ func (wr *Wrangler) emergencyReparentShardLocked(ctx context.Context, ev *events // Check corner cases we're going to depend on masterElectTabletInfo, ok := tabletMap[*masterElectTabletAlias] if !ok { - return fmt.Errorf("master-elect tablet %v is not in the shard", topo.TabletAliasString(masterElectTabletAlias)) + return fmt.Errorf("master-elect tablet %v is not in the shard", topoproto.TabletAliasString(masterElectTabletAlias)) } ev.NewMaster = *masterElectTabletInfo.Tablet - if topo.TabletAliasEqual(shardInfo.MasterAlias, masterElectTabletAlias) { - return fmt.Errorf("master-elect tablet %v is already the master", topo.TabletAliasString(masterElectTabletAlias)) + if topoproto.TabletAliasEqual(shardInfo.MasterAlias, masterElectTabletAlias) { + return fmt.Errorf("master-elect tablet %v is already the master", topoproto.TabletAliasString(masterElectTabletAlias)) } // Deal with the old master: try to remote-scrap it, if it's // truely dead we force-scrap it. Remove it from our map in any case. - if !topo.TabletAliasIsZero(shardInfo.MasterAlias) { + if shardInfo.HasMaster() { scrapOldMaster := true oldMasterTabletInfo, ok := tabletMap[*shardInfo.MasterAlias] if ok { @@ -501,14 +502,14 @@ func (wr *Wrangler) emergencyReparentShardLocked(ctx context.Context, ev *events } else { oldMasterTabletInfo, err = wr.ts.GetTablet(ctx, shardInfo.MasterAlias) if err != nil { - wr.logger.Warningf("cannot read old master tablet %v, won't touch it: %v", topo.TabletAliasString(shardInfo.MasterAlias), err) + wr.logger.Warningf("cannot read old master tablet %v, won't touch it: %v", topoproto.TabletAliasString(shardInfo.MasterAlias), err) scrapOldMaster = false } } if scrapOldMaster { ev.OldMaster = *oldMasterTabletInfo.Tablet - wr.logger.Infof("scrapping old master %v", topo.TabletAliasString(shardInfo.MasterAlias)) + wr.logger.Infof("scrapping old master %v", topoproto.TabletAliasString(shardInfo.MasterAlias)) ctx, cancel := context.WithTimeout(ctx, waitSlaveTimeout) defer cancel() @@ -533,12 +534,12 @@ func (wr *Wrangler) emergencyReparentShardLocked(ctx context.Context, ev *events wg.Add(1) go func(alias pb.TabletAlias, tabletInfo *topo.TabletInfo) { defer wg.Done() - wr.logger.Infof("getting replication position from %v", topo.TabletAliasString(&alias)) + wr.logger.Infof("getting replication position from %v", topoproto.TabletAliasString(&alias)) ctx, cancel := context.WithTimeout(ctx, waitSlaveTimeout) defer cancel() rp, err := wr.TabletManagerClient().StopReplicationAndGetStatus(ctx, tabletInfo) if err != nil { - wr.logger.Warningf("failed to get replication status from %v, ignoring tablet: %v", topo.TabletAliasString(&alias), err) + wr.logger.Warningf("failed to get replication status from %v, ignoring tablet: %v", topoproto.TabletAliasString(&alias), err) return } mu.Lock() @@ -551,23 +552,23 @@ func (wr *Wrangler) emergencyReparentShardLocked(ctx context.Context, ev *events // Verify masterElect is alive and has the most advanced position masterElectStatus, ok := statusMap[*masterElectTabletAlias] if !ok { - return fmt.Errorf("couldn't get master elect %v replication position", topo.TabletAliasString(masterElectTabletAlias)) + return fmt.Errorf("couldn't get master elect %v replication position", topoproto.TabletAliasString(masterElectTabletAlias)) } for alias, status := range statusMap { - if topo.TabletAliasEqual(&alias, masterElectTabletAlias) { + if topoproto.TabletAliasEqual(&alias, masterElectTabletAlias) { continue } if !masterElectStatus.Position.AtLeast(status.Position) { - return fmt.Errorf("tablet %v is more advanced than master elect tablet %v: %v > %v", topo.TabletAliasString(&alias), topo.TabletAliasString(masterElectTabletAlias), status.Position, masterElectStatus) + return fmt.Errorf("tablet %v is more advanced than master elect tablet %v: %v > %v", topoproto.TabletAliasString(&alias), topoproto.TabletAliasString(masterElectTabletAlias), status.Position, masterElectStatus) } } // Promote the masterElect - wr.logger.Infof("promote slave %v", topo.TabletAliasString(masterElectTabletAlias)) + wr.logger.Infof("promote slave %v", topoproto.TabletAliasString(masterElectTabletAlias)) event.DispatchUpdate(ev, "promoting slave") rp, err := wr.tmc.PromoteSlave(ctx, masterElectTabletInfo) if err != nil { - return fmt.Errorf("master-elect tablet %v failed to be upgraded to master: %v", topo.TabletAliasString(masterElectTabletAlias), err) + return fmt.Errorf("master-elect tablet %v failed to be upgraded to master: %v", topoproto.TabletAliasString(masterElectTabletAlias), err) } // Reset replication on all slaves to point to the new master, and @@ -582,24 +583,24 @@ func (wr *Wrangler) emergencyReparentShardLocked(ctx context.Context, ev *events rec := concurrency.AllErrorRecorder{} var masterErr error for alias, tabletInfo := range tabletMap { - if topo.TabletAliasEqual(&alias, masterElectTabletAlias) { + if topoproto.TabletAliasEqual(&alias, masterElectTabletAlias) { wgMaster.Add(1) go func(alias pb.TabletAlias, tabletInfo *topo.TabletInfo) { defer wgMaster.Done() - wr.logger.Infof("populating reparent journal on new master %v", topo.TabletAliasString(&alias)) + wr.logger.Infof("populating reparent journal on new master %v", topoproto.TabletAliasString(&alias)) masterErr = wr.TabletManagerClient().PopulateReparentJournal(ctx, tabletInfo, now, emergencyReparentShardOperation, &alias, rp) }(alias, tabletInfo) } else { wgSlaves.Add(1) go func(alias pb.TabletAlias, tabletInfo *topo.TabletInfo) { defer wgSlaves.Done() - wr.logger.Infof("setting new master on slave %v", topo.TabletAliasString(&alias)) + wr.logger.Infof("setting new master on slave %v", topoproto.TabletAliasString(&alias)) forceStartSlave := false if status, ok := statusMap[alias]; ok { forceStartSlave = status.SlaveIORunning || status.SlaveSQLRunning } if err := wr.TabletManagerClient().SetMaster(ctx, tabletInfo, masterElectTabletAlias, now, forceStartSlave); err != nil { - rec.RecordError(fmt.Errorf("Tablet %v SetMaster failed: %v", topo.TabletAliasString(&alias), err)) + rec.RecordError(fmt.Errorf("Tablet %v SetMaster failed: %v", topoproto.TabletAliasString(&alias), err)) } }(alias, tabletInfo) } @@ -612,7 +613,7 @@ func (wr *Wrangler) emergencyReparentShardLocked(ctx context.Context, ev *events wgSlaves.Wait() return fmt.Errorf("failed to PopulateReparentJournal on master: %v", masterErr) } - wr.logger.Infof("updating shard record with new master %v", topo.TabletAliasString(masterElectTabletAlias)) + wr.logger.Infof("updating shard record with new master %v", topoproto.TabletAliasString(masterElectTabletAlias)) shardInfo.MasterAlias = masterElectTabletAlias if err := topo.UpdateShard(ctx, wr.ts, shardInfo); err != nil { wgSlaves.Wait() diff --git a/go/vt/wrangler/schema.go b/go/vt/wrangler/schema.go index 6a2373ad2b..e85ca0e9bd 100644 --- a/go/vt/wrangler/schema.go +++ b/go/vt/wrangler/schema.go @@ -21,6 +21,7 @@ import ( "github.com/youtube/vitess/go/vt/schemamanager" "github.com/youtube/vitess/go/vt/tabletmanager/actionnode" "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "github.com/youtube/vitess/go/vt/topotools/events" pb "github.com/youtube/vitess/go/vt/proto/topodata" @@ -49,15 +50,15 @@ func (wr *Wrangler) ReloadSchema(ctx context.Context, tabletAlias *pb.TabletAlia // helper method to asynchronously diff a schema func (wr *Wrangler) diffSchema(ctx context.Context, masterSchema *myproto.SchemaDefinition, masterTabletAlias, alias *pb.TabletAlias, excludeTables []string, includeViews bool, wg *sync.WaitGroup, er concurrency.ErrorRecorder) { defer wg.Done() - log.Infof("Gathering schema for %v", topo.TabletAliasString(alias)) + log.Infof("Gathering schema for %v", topoproto.TabletAliasString(alias)) slaveSchema, err := wr.GetSchema(ctx, alias, nil, excludeTables, includeViews) if err != nil { er.RecordError(err) return } - log.Infof("Diffing schema for %v", topo.TabletAliasString(alias)) - myproto.DiffSchema(topo.TabletAliasString(masterTabletAlias), masterSchema, topo.TabletAliasString(alias), slaveSchema, er) + log.Infof("Diffing schema for %v", topoproto.TabletAliasString(alias)) + myproto.DiffSchema(topoproto.TabletAliasString(masterTabletAlias), masterSchema, topoproto.TabletAliasString(alias), slaveSchema, er) } // ValidateSchemaShard will diff the schema from all the tablets in the shard. @@ -68,10 +69,10 @@ func (wr *Wrangler) ValidateSchemaShard(ctx context.Context, keyspace, shard str } // get schema from the master, or error - if topo.TabletAliasIsZero(si.MasterAlias) { + if !si.HasMaster() { return fmt.Errorf("No master in shard %v/%v", keyspace, shard) } - log.Infof("Gathering schema for master %v", topo.TabletAliasString(si.MasterAlias)) + log.Infof("Gathering schema for master %v", topoproto.TabletAliasString(si.MasterAlias)) masterSchema, err := wr.GetSchema(ctx, si.MasterAlias, nil, excludeTables, includeViews) if err != nil { return err @@ -88,7 +89,7 @@ func (wr *Wrangler) ValidateSchemaShard(ctx context.Context, keyspace, shard str er := concurrency.AllErrorRecorder{} wg := sync.WaitGroup{} for _, alias := range aliases { - if topo.TabletAliasEqual(alias, si.MasterAlias) { + if topoproto.TabletAliasEqual(alias, si.MasterAlias) { continue } @@ -125,11 +126,11 @@ func (wr *Wrangler) ValidateSchemaKeyspace(ctx context.Context, keyspace string, if err != nil { return err } - if topo.TabletAliasIsZero(si.MasterAlias) { + if !si.HasMaster() { return fmt.Errorf("No master in shard %v/%v", keyspace, shards[0]) } referenceAlias := si.MasterAlias - log.Infof("Gathering schema for reference master %v", topo.TabletAliasString(referenceAlias)) + log.Infof("Gathering schema for reference master %v", topoproto.TabletAliasString(referenceAlias)) referenceSchema, err := wr.GetSchema(ctx, referenceAlias, nil, excludeTables, includeViews) if err != nil { return err @@ -146,7 +147,7 @@ func (wr *Wrangler) ValidateSchemaKeyspace(ctx context.Context, keyspace string, } for _, alias := range aliases { - if topo.TabletAliasEqual(alias, si.MasterAlias) { + if topoproto.TabletAliasEqual(alias, si.MasterAlias) { continue } @@ -162,7 +163,7 @@ func (wr *Wrangler) ValidateSchemaKeyspace(ctx context.Context, keyspace string, continue } - if topo.TabletAliasIsZero(si.MasterAlias) { + if !si.HasMaster() { er.RecordError(fmt.Errorf("No master in shard %v/%v", keyspace, shard)) continue } @@ -307,7 +308,7 @@ func (wr *Wrangler) applySchemaShardSimple(ctx context.Context, statusArray []*t // BeforeSchema. If not, we shouldn't proceed log.Infof("Checking schema on all tablets") for _, status := range statusArray { - diffs := myproto.DiffSchemaToArray("master", preflight.BeforeSchema, topo.TabletAliasString(status.ti.Alias), status.beforeSchema) + diffs := myproto.DiffSchemaToArray("master", preflight.BeforeSchema, topoproto.TabletAliasString(status.ti.Alias), status.beforeSchema) if len(diffs) > 0 { if force { log.Warningf("Tablet %v has inconsistent schema, ignoring: %v", status.ti.AliasString(), strings.Join(diffs, "\n")) @@ -327,14 +328,14 @@ func (wr *Wrangler) applySchemaShardComplex(ctx context.Context, statusArray []* // apply the schema change to all replica / slave tablets for _, status := range statusArray { // if already applied, we skip this guy - diffs := myproto.DiffSchemaToArray("after", preflight.AfterSchema, topo.TabletAliasString(status.ti.Alias), status.beforeSchema) + diffs := myproto.DiffSchemaToArray("after", preflight.AfterSchema, topoproto.TabletAliasString(status.ti.Alias), status.beforeSchema) if len(diffs) == 0 { log.Infof("Tablet %v already has the AfterSchema, skipping", status.ti.AliasString()) continue } // make sure the before schema matches - diffs = myproto.DiffSchemaToArray("master", preflight.BeforeSchema, topo.TabletAliasString(status.ti.Alias), status.beforeSchema) + diffs = myproto.DiffSchemaToArray("master", preflight.BeforeSchema, topoproto.TabletAliasString(status.ti.Alias), status.beforeSchema) if len(diffs) > 0 { if force { log.Warningf("Tablet %v has inconsistent schema, ignoring: %v", status.ti.AliasString(), strings.Join(diffs, "\n")) @@ -375,8 +376,8 @@ func (wr *Wrangler) applySchemaShardComplex(ctx context.Context, statusArray []* } // if newParentTabletAlias is passed in, use that as the new master - if !topo.TabletAliasIsZero(newParentTabletAlias) { - log.Infof("Reparenting with new master set to %v", topo.TabletAliasString(newParentTabletAlias)) + if !topoproto.TabletAliasIsZero(newParentTabletAlias) { + log.Infof("Reparenting with new master set to %v", topoproto.TabletAliasString(newParentTabletAlias)) oldMasterAlias := shardInfo.MasterAlias // Create reusable Reparent event with available info @@ -391,7 +392,7 @@ func (wr *Wrangler) applySchemaShardComplex(ctx context.Context, statusArray []* // with the previous implementation of the reparent. // (this code will be refactored at some point anyway) if err := wr.Scrap(ctx, oldMasterAlias, false, false); err != nil { - wr.Logger().Warningf("Scrapping old master %v from shard %v/%v failed: %v", topo.TabletAliasString(oldMasterAlias), shardInfo.Keyspace(), shardInfo.ShardName(), err) + wr.Logger().Warningf("Scrapping old master %v from shard %v/%v failed: %v", topoproto.TabletAliasString(oldMasterAlias), shardInfo.Keyspace(), shardInfo.ShardName(), err) } } return &myproto.SchemaChangeResult{BeforeSchema: preflight.BeforeSchema, AfterSchema: preflight.AfterSchema}, nil diff --git a/go/vt/wrangler/shard.go b/go/vt/wrangler/shard.go index 63ab382587..60ce01fbc3 100644 --- a/go/vt/wrangler/shard.go +++ b/go/vt/wrangler/shard.go @@ -9,6 +9,7 @@ import ( "github.com/youtube/vitess/go/vt/tabletmanager/actionnode" "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "golang.org/x/net/context" pb "github.com/youtube/vitess/go/vt/proto/topodata" @@ -36,7 +37,7 @@ func (wr *Wrangler) updateShardCellsAndMaster(ctx context.Context, si *topo.Shar if !si.HasCell(tabletAlias.Cell) { shardUpdateRequired = true } - if tabletType == pb.TabletType_MASTER && !topo.TabletAliasEqual(si.MasterAlias, tabletAlias) { + if tabletType == pb.TabletType_MASTER && !topoproto.TabletAliasEqual(si.MasterAlias, tabletAlias) { shardUpdateRequired = true } if !shardUpdateRequired { @@ -63,9 +64,9 @@ func (wr *Wrangler) updateShardCellsAndMaster(ctx context.Context, si *topo.Shar si.Cells = append(si.Cells, tabletAlias.Cell) wasUpdated = true } - if tabletType == pb.TabletType_MASTER && !topo.TabletAliasEqual(si.MasterAlias, tabletAlias) { - if !topo.TabletAliasIsZero(si.MasterAlias) && !force { - return wr.unlockShard(ctx, keyspace, shard, actionNode, lockPath, fmt.Errorf("creating this tablet would override old master %v in shard %v/%v", topo.TabletAliasString(si.MasterAlias), keyspace, shard)) + if tabletType == pb.TabletType_MASTER && !topoproto.TabletAliasEqual(si.MasterAlias, tabletAlias) { + if si.HasMaster() && !force { + return wr.unlockShard(ctx, keyspace, shard, actionNode, lockPath, fmt.Errorf("creating this tablet would override old master %v in shard %v/%v", topoproto.TabletAliasString(si.MasterAlias), keyspace, shard)) } si.MasterAlias = tabletAlias wasUpdated = true @@ -167,7 +168,7 @@ func (wr *Wrangler) DeleteShard(ctx context.Context, keyspace, shard string, rec for tabletAlias := range tabletMap { // We don't care about scrapping or updating the replication graph, // because we're about to delete the entire replication graph. - wr.Logger().Infof("Deleting tablet %v", topo.TabletAliasString(&tabletAlias)) + wr.Logger().Infof("Deleting tablet %v", topoproto.TabletAliasString(&tabletAlias)) if err := wr.TopoServer().DeleteTablet(ctx, &tabletAlias); err != nil && err != topo.ErrNoNode { // Unlike the errors below in non-recursive steps, we don't want to // continue if a DeleteTablet fails. If we continue and delete the @@ -176,7 +177,7 @@ func (wr *Wrangler) DeleteShard(ctx context.Context, keyspace, shard string, rec // // If the problem is temporary, or resolved externally, re-running // DeleteShard will skip over tablets that were already deleted. - return fmt.Errorf("can't delete tablet %v: %v", topo.TabletAliasString(&tabletAlias), err) + return fmt.Errorf("can't delete tablet %v: %v", topoproto.TabletAliasString(&tabletAlias), err) } } } else if len(tabletMap) > 0 { @@ -189,7 +190,7 @@ func (wr *Wrangler) DeleteShard(ctx context.Context, keyspace, shard string, rec wr.Logger().Warningf("Cannot delete ShardReplication in cell %v for %v/%v: %v", cell, keyspace, shard, err) } - for _, t := range topo.AllTabletTypes { + for _, t := range topoproto.AllTabletTypes { if !topo.IsInServingGraph(t) { continue } @@ -240,7 +241,7 @@ func (wr *Wrangler) removeShardCell(ctx context.Context, keyspace, shard, cell s // check the master alias is not in the cell if shardInfo.MasterAlias != nil && shardInfo.MasterAlias.Cell == cell { - return fmt.Errorf("master %v is in the cell '%v' we want to remove", topo.TabletAliasString(shardInfo.MasterAlias), cell) + return fmt.Errorf("master %v is in the cell '%v' we want to remove", topoproto.TabletAliasString(shardInfo.MasterAlias), cell) } // get the ShardReplication object in the cell @@ -252,9 +253,9 @@ func (wr *Wrangler) removeShardCell(ctx context.Context, keyspace, shard, cell s for _, node := range sri.Nodes { // We don't care about scrapping or updating the replication graph, // because we're about to delete the entire replication graph. - wr.Logger().Infof("Deleting tablet %v", topo.TabletAliasString(node.TabletAlias)) + wr.Logger().Infof("Deleting tablet %v", topoproto.TabletAliasString(node.TabletAlias)) if err := wr.TopoServer().DeleteTablet(ctx, node.TabletAlias); err != nil && err != topo.ErrNoNode { - return fmt.Errorf("can't delete tablet %v: %v", topo.TabletAliasString(node.TabletAlias), err) + return fmt.Errorf("can't delete tablet %v: %v", topoproto.TabletAliasString(node.TabletAlias), err) } } } else if len(sri.Nodes) > 0 { diff --git a/go/vt/wrangler/tablet.go b/go/vt/wrangler/tablet.go index 9afe43ab47..3604aab2f1 100644 --- a/go/vt/wrangler/tablet.go +++ b/go/vt/wrangler/tablet.go @@ -11,6 +11,7 @@ import ( "github.com/youtube/vitess/go/vt/key" "github.com/youtube/vitess/go/vt/tabletmanager/actionnode" "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "github.com/youtube/vitess/go/vt/topotools" "golang.org/x/net/context" @@ -53,8 +54,8 @@ func (wr *Wrangler) InitTablet(ctx context.Context, tablet *pb.Tablet, force, cr if !key.KeyRangeEqual(si.KeyRange, tablet.KeyRange) { return fmt.Errorf("shard %v/%v has a different KeyRange: %v != %v", tablet.Keyspace, tablet.Shard, si.KeyRange, tablet.KeyRange) } - if tablet.Type == pb.TabletType_MASTER && !topo.TabletAliasIsZero(si.MasterAlias) && !topo.TabletAliasEqual(si.MasterAlias, tablet.Alias) && !force { - return fmt.Errorf("creating this tablet would override old master %v in shard %v/%v", topo.TabletAliasString(si.MasterAlias), tablet.Keyspace, tablet.Shard) + if tablet.Type == pb.TabletType_MASTER && si.HasMaster() && !topoproto.TabletAliasEqual(si.MasterAlias, tablet.Alias) && !force { + return fmt.Errorf("creating this tablet would override old master %v in shard %v/%v", topoproto.TabletAliasString(si.MasterAlias), tablet.Keyspace, tablet.Shard) } // update the shard record if needed @@ -69,12 +70,12 @@ func (wr *Wrangler) InitTablet(ctx context.Context, tablet *pb.Tablet, force, cr if update || force { oldTablet, err := wr.ts.GetTablet(ctx, tablet.Alias) if err != nil { - wr.Logger().Warningf("failed reading tablet %v: %v", topo.TabletAliasString(tablet.Alias), err) + wr.Logger().Warningf("failed reading tablet %v: %v", topoproto.TabletAliasString(tablet.Alias), err) } else { if oldTablet.Keyspace == tablet.Keyspace && oldTablet.Shard == tablet.Shard { *(oldTablet.Tablet) = *tablet if err := topo.UpdateTablet(ctx, wr.ts, oldTablet); err != nil { - wr.Logger().Warningf("failed updating tablet %v: %v", topo.TabletAliasString(tablet.Alias), err) + wr.Logger().Warningf("failed updating tablet %v: %v", topoproto.TabletAliasString(tablet.Alias), err) // now fall through the Scrap case } else { if !topo.IsInReplicationGraph(tablet.Type) { @@ -82,7 +83,7 @@ func (wr *Wrangler) InitTablet(ctx context.Context, tablet *pb.Tablet, force, cr } if err := topo.UpdateTabletReplicationData(ctx, wr.ts, tablet); err != nil { - wr.Logger().Warningf("failed updating tablet replication data for %v: %v", topo.TabletAliasString(tablet.Alias), err) + wr.Logger().Warningf("failed updating tablet replication data for %v: %v", topoproto.TabletAliasString(tablet.Alias), err) // now fall through the Scrap case } else { return nil @@ -93,12 +94,12 @@ func (wr *Wrangler) InitTablet(ctx context.Context, tablet *pb.Tablet, force, cr } if force { if err = wr.Scrap(ctx, tablet.Alias, force, false); err != nil { - wr.Logger().Errorf("failed scrapping tablet %v: %v", topo.TabletAliasString(tablet.Alias), err) + wr.Logger().Errorf("failed scrapping tablet %v: %v", topoproto.TabletAliasString(tablet.Alias), err) return err } if err := wr.ts.DeleteTablet(ctx, tablet.Alias); err != nil { // we ignore this - wr.Logger().Errorf("failed deleting tablet %v: %v", topo.TabletAliasString(tablet.Alias), err) + wr.Logger().Errorf("failed deleting tablet %v: %v", topoproto.TabletAliasString(tablet.Alias), err) } return topo.CreateTablet(ctx, wr.ts, tablet) } @@ -153,7 +154,7 @@ func (wr *Wrangler) Scrap(ctx context.Context, tabletAlias *pb.TabletAlias, forc } // update it if the right alias is there - if topo.TabletAliasEqual(si.MasterAlias, tabletAlias) { + if topoproto.TabletAliasEqual(si.MasterAlias, tabletAlias) { si.MasterAlias = nil // write it back @@ -161,7 +162,7 @@ func (wr *Wrangler) Scrap(ctx context.Context, tabletAlias *pb.TabletAlias, forc return wr.unlockShard(ctx, ti.Keyspace, ti.Shard, actionNode, lockPath, err) } } else { - wr.Logger().Warningf("Scrapping master %v from shard %v/%v but master in Shard object was %v", topo.TabletAliasString(tabletAlias), ti.Keyspace, ti.Shard, si.MasterAlias) + wr.Logger().Warningf("Scrapping master %v from shard %v/%v but master in Shard object was %v", topoproto.TabletAliasString(tabletAlias), ti.Keyspace, ti.Shard, si.MasterAlias) } // and unlock diff --git a/go/vt/wrangler/testlib/backup_test.go b/go/vt/wrangler/testlib/backup_test.go index 42fd062ae9..197b03f0aa 100644 --- a/go/vt/wrangler/testlib/backup_test.go +++ b/go/vt/wrangler/testlib/backup_test.go @@ -18,7 +18,7 @@ import ( "github.com/youtube/vitess/go/vt/mysqlctl/filebackupstorage" myproto "github.com/youtube/vitess/go/vt/mysqlctl/proto" "github.com/youtube/vitess/go/vt/tabletmanager/tmclient" - "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "github.com/youtube/vitess/go/vt/wrangler" "github.com/youtube/vitess/go/vt/zktopo" "golang.org/x/net/context" @@ -93,7 +93,7 @@ func TestBackupRestore(t *testing.T) { defer sourceTablet.StopActionLoop(t) // run the backup - if err := vp.Run([]string{"Backup", topo.TabletAliasString(sourceTablet.Tablet.Alias)}); err != nil { + if err := vp.Run([]string{"Backup", topoproto.TabletAliasString(sourceTablet.Tablet.Alias)}); err != nil { t.Fatalf("Backup failed: %v", err) } diff --git a/go/vt/wrangler/testlib/copy_schema_shard_test.go b/go/vt/wrangler/testlib/copy_schema_shard_test.go index 7da3ee1061..01a34b4f9d 100644 --- a/go/vt/wrangler/testlib/copy_schema_shard_test.go +++ b/go/vt/wrangler/testlib/copy_schema_shard_test.go @@ -13,7 +13,7 @@ import ( "github.com/youtube/vitess/go/vt/logutil" myproto "github.com/youtube/vitess/go/vt/mysqlctl/proto" "github.com/youtube/vitess/go/vt/tabletmanager/tmclient" - "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "github.com/youtube/vitess/go/vt/vttest/fakesqldb" "github.com/youtube/vitess/go/vt/wrangler" "github.com/youtube/vitess/go/vt/zktopo" @@ -156,7 +156,7 @@ func copySchema(t *testing.T, useShardAsSource bool) { db.AddQuery(createTable, &mproto.QueryResult{}) db.AddQuery(createTableView, &mproto.QueryResult{}) - source := topo.TabletAliasString(sourceRdonly.Tablet.Alias) + source := topoproto.TabletAliasString(sourceRdonly.Tablet.Alias) if useShardAsSource { source = "ks/-80" } diff --git a/go/vt/wrangler/testlib/emergency_reparent_shard_test.go b/go/vt/wrangler/testlib/emergency_reparent_shard_test.go index 3ccf4c02a9..6dbafdf6ee 100644 --- a/go/vt/wrangler/testlib/emergency_reparent_shard_test.go +++ b/go/vt/wrangler/testlib/emergency_reparent_shard_test.go @@ -13,7 +13,7 @@ import ( "github.com/youtube/vitess/go/vt/logutil" myproto "github.com/youtube/vitess/go/vt/mysqlctl/proto" "github.com/youtube/vitess/go/vt/tabletmanager/tmclient" - "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "github.com/youtube/vitess/go/vt/wrangler" "github.com/youtube/vitess/go/vt/zktopo" "golang.org/x/net/context" @@ -103,7 +103,7 @@ func TestEmergencyReparentShard(t *testing.T) { defer goodSlave2.StopActionLoop(t) // run EmergencyReparentShard - if err := vp.Run([]string{"EmergencyReparentShard", "-wait_slave_timeout", "10s", newMaster.Tablet.Keyspace + "/" + newMaster.Tablet.Shard, topo.TabletAliasString(newMaster.Tablet.Alias)}); err != nil { + if err := vp.Run([]string{"EmergencyReparentShard", "-wait_slave_timeout", "10s", newMaster.Tablet.Keyspace + "/" + newMaster.Tablet.Shard, topoproto.TabletAliasString(newMaster.Tablet.Alias)}); err != nil { t.Fatalf("EmergencyReparentShard failed: %v", err) } diff --git a/go/vt/wrangler/testlib/init_shard_master_test.go b/go/vt/wrangler/testlib/init_shard_master_test.go index 0da5559b7c..b9cb7a2312 100644 --- a/go/vt/wrangler/testlib/init_shard_master_test.go +++ b/go/vt/wrangler/testlib/init_shard_master_test.go @@ -13,6 +13,7 @@ import ( myproto "github.com/youtube/vitess/go/vt/mysqlctl/proto" "github.com/youtube/vitess/go/vt/tabletmanager/tmclient" "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "github.com/youtube/vitess/go/vt/wrangler" "github.com/youtube/vitess/go/vt/zktopo" "golang.org/x/net/context" @@ -93,7 +94,7 @@ func TestInitMasterShard(t *testing.T) { defer goodSlave2.StopActionLoop(t) // run InitShardMaster - if err := vp.Run([]string{"InitShardMaster", "-wait_slave_timeout", "10s", master.Tablet.Keyspace + "/" + master.Tablet.Shard, topo.TabletAliasString(master.Tablet.Alias)}); err != nil { + if err := vp.Run([]string{"InitShardMaster", "-wait_slave_timeout", "10s", master.Tablet.Keyspace + "/" + master.Tablet.Shard, topoproto.TabletAliasString(master.Tablet.Alias)}); err != nil { t.Fatalf("InitShardMaster failed: %v", err) } @@ -105,7 +106,7 @@ func TestInitMasterShard(t *testing.T) { if err != nil { t.Fatalf("GetShard failed: %v", err) } - if !topo.TabletAliasEqual(si.MasterAlias, master.Tablet.Alias) { + if !topoproto.TabletAliasEqual(si.MasterAlias, master.Tablet.Alias) { t.Errorf("unexpected shard master alias, got %v expected %v", si.MasterAlias, master.Tablet.Alias) } if err := master.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { @@ -242,7 +243,7 @@ func TestInitMasterShardOneSlaveFails(t *testing.T) { if err != nil { t.Fatalf("GetShard failed: %v", err) } - if !topo.TabletAliasEqual(si.MasterAlias, master.Tablet.Alias) { + if !topoproto.TabletAliasEqual(si.MasterAlias, master.Tablet.Alias) { t.Errorf("unexpected shard master alias, got %v expected %v", si.MasterAlias, master.Tablet.Alias) } } diff --git a/go/vt/wrangler/testlib/planned_reparent_shard_test.go b/go/vt/wrangler/testlib/planned_reparent_shard_test.go index 9caadaa570..e334f4391f 100644 --- a/go/vt/wrangler/testlib/planned_reparent_shard_test.go +++ b/go/vt/wrangler/testlib/planned_reparent_shard_test.go @@ -13,7 +13,7 @@ import ( myproto "github.com/youtube/vitess/go/vt/mysqlctl/proto" "github.com/youtube/vitess/go/vt/tabletmanager/tmclient" "github.com/youtube/vitess/go/vt/tabletserver" - "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "github.com/youtube/vitess/go/vt/wrangler" "github.com/youtube/vitess/go/vt/zktopo" @@ -96,7 +96,7 @@ func TestPlannedReparentShard(t *testing.T) { defer goodSlave2.StopActionLoop(t) // run PlannedReparentShard - if err := vp.Run([]string{"PlannedReparentShard", "-wait_slave_timeout", "10s", newMaster.Tablet.Keyspace + "/" + newMaster.Tablet.Shard, topo.TabletAliasString(newMaster.Tablet.Alias)}); err != nil { + if err := vp.Run([]string{"PlannedReparentShard", "-wait_slave_timeout", "10s", newMaster.Tablet.Keyspace + "/" + newMaster.Tablet.Shard, topoproto.TabletAliasString(newMaster.Tablet.Alias)}); err != nil { t.Fatalf("PlannedReparentShard failed: %v", err) } diff --git a/go/vt/wrangler/testlib/reparent_external_test.go b/go/vt/wrangler/testlib/reparent_external_test.go index 1653ce8408..a2b5734c8e 100644 --- a/go/vt/wrangler/testlib/reparent_external_test.go +++ b/go/vt/wrangler/testlib/reparent_external_test.go @@ -17,6 +17,7 @@ import ( "github.com/youtube/vitess/go/vt/tabletmanager" "github.com/youtube/vitess/go/vt/tabletmanager/tmclient" "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "github.com/youtube/vitess/go/vt/topotools" "github.com/youtube/vitess/go/vt/topotools/events" "github.com/youtube/vitess/go/vt/wrangler" @@ -59,11 +60,11 @@ func TestTabletExternallyReparented(t *testing.T) { t.Fatalf("GetTabletMapForShardByCell should have worked but got: %v", err) } master, err := topotools.FindTabletByIPAddrAndPort(tabletMap, oldMaster.Tablet.Ip, "vt", oldMaster.Tablet.PortMap["vt"]) - if err != nil || !topo.TabletAliasEqual(&master, oldMaster.Tablet.Alias) { + if err != nil || !topoproto.TabletAliasEqual(&master, oldMaster.Tablet.Alias) { t.Fatalf("FindTabletByIPAddrAndPort(master) failed: %v %v", err, master) } slave1, err := topotools.FindTabletByIPAddrAndPort(tabletMap, goodSlave1.Tablet.Ip, "vt", goodSlave1.Tablet.PortMap["vt"]) - if err != nil || !topo.TabletAliasEqual(&slave1, goodSlave1.Tablet.Alias) { + if err != nil || !topoproto.TabletAliasEqual(&slave1, goodSlave1.Tablet.Alias) { t.Fatalf("FindTabletByIPAddrAndPort(slave1) failed: %v %v", err, master) } slave2, err := topotools.FindTabletByIPAddrAndPort(tabletMap, goodSlave2.Tablet.Ip, "vt", goodSlave2.Tablet.PortMap["vt"]) @@ -83,7 +84,7 @@ func TestTabletExternallyReparented(t *testing.T) { t.Fatalf("GetTabletMapForShard should have returned ErrPartialResult but got: %v", err) } master, err = topotools.FindTabletByIPAddrAndPort(tabletMap, oldMaster.Tablet.Ip, "vt", oldMaster.Tablet.PortMap["vt"]) - if err != nil || !topo.TabletAliasEqual(&master, oldMaster.Tablet.Alias) { + if err != nil || !topoproto.TabletAliasEqual(&master, oldMaster.Tablet.Alias) { t.Fatalf("FindTabletByIPAddrAndPort(master) failed: %v %v", err, master) } @@ -117,7 +118,7 @@ func TestTabletExternallyReparented(t *testing.T) { if err != nil { t.Fatalf("GetTablet failed: %v", err) } - if err := vp.Run([]string{"TabletExternallyReparented", topo.TabletAliasString(oldMaster.Tablet.Alias)}); err != nil { + if err := vp.Run([]string{"TabletExternallyReparented", topoproto.TabletAliasString(oldMaster.Tablet.Alias)}); err != nil { t.Fatalf("TabletExternallyReparented(same master) should have worked: %v", err) } diff --git a/go/vt/wrangler/testlib/reparent_utils_test.go b/go/vt/wrangler/testlib/reparent_utils_test.go index 56d109b635..0e9d480fa9 100644 --- a/go/vt/wrangler/testlib/reparent_utils_test.go +++ b/go/vt/wrangler/testlib/reparent_utils_test.go @@ -13,6 +13,7 @@ import ( myproto "github.com/youtube/vitess/go/vt/mysqlctl/proto" "github.com/youtube/vitess/go/vt/tabletmanager/tmclient" "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "github.com/youtube/vitess/go/vt/wrangler" "github.com/youtube/vitess/go/vt/zktopo" "golang.org/x/net/context" @@ -76,12 +77,12 @@ func TestShardReplicationStatuses(t *testing.T) { if len(ti) != 2 || len(rs) != 2 { t.Fatalf("ShardReplicationStatuses returned wrong results: %v %v", ti, rs) } - if topo.TabletAliasEqual(ti[0].Alias, slave.Tablet.Alias) { + if topoproto.TabletAliasEqual(ti[0].Alias, slave.Tablet.Alias) { ti[0], ti[1] = ti[1], ti[0] rs[0], rs[1] = rs[1], rs[0] } - if !topo.TabletAliasEqual(ti[0].Alias, master.Tablet.Alias) || - !topo.TabletAliasEqual(ti[1].Alias, slave.Tablet.Alias) || + if !topoproto.TabletAliasEqual(ti[0].Alias, master.Tablet.Alias) || + !topoproto.TabletAliasEqual(ti[1].Alias, slave.Tablet.Alias) || rs[0].MasterHost != "" || rs[1].MasterHost != master.Tablet.Hostname { t.Fatalf("ShardReplicationStatuses returend wrong results: %v %v", ti, rs) diff --git a/go/vt/wrangler/validator.go b/go/vt/wrangler/validator.go index 4ace230453..5aa8e4fbe0 100644 --- a/go/vt/wrangler/validator.go +++ b/go/vt/wrangler/validator.go @@ -11,6 +11,7 @@ import ( "sync" "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "golang.org/x/net/context" pb "github.com/youtube/vitess/go/vt/proto/topodata" @@ -82,9 +83,9 @@ func (wr *Wrangler) validateAllTablets(ctx context.Context, wg *sync.WaitGroup, go func(alias *pb.TabletAlias) { defer wg.Done() if err := topo.Validate(ctx, wr.ts, alias); err != nil { - results <- fmt.Errorf("Validate(%v) failed: %v", topo.TabletAliasString(alias), err) + results <- fmt.Errorf("Validate(%v) failed: %v", topoproto.TabletAliasString(alias), err) } else { - wr.Logger().Infof("tablet %v is valid", topo.TabletAliasString(alias)) + wr.Logger().Infof("tablet %v is valid", topoproto.TabletAliasString(alias)) } }(alias) } @@ -128,12 +129,12 @@ func (wr *Wrangler) validateShard(ctx context.Context, keyspace, shard string, p for _, alias := range aliases { tabletInfo, ok := tabletMap[*alias] if !ok { - results <- fmt.Errorf("tablet %v not found in map", topo.TabletAliasString(alias)) + results <- fmt.Errorf("tablet %v not found in map", topoproto.TabletAliasString(alias)) continue } if tabletInfo.Type == pb.TabletType_MASTER { if masterAlias != nil { - results <- fmt.Errorf("shard %v/%v already has master %v but found other master %v", keyspace, shard, topo.TabletAliasString(masterAlias), topo.TabletAliasString(alias)) + results <- fmt.Errorf("shard %v/%v already has master %v but found other master %v", keyspace, shard, topoproto.TabletAliasString(masterAlias), topoproto.TabletAliasString(alias)) } else { masterAlias = alias } @@ -142,8 +143,8 @@ func (wr *Wrangler) validateShard(ctx context.Context, keyspace, shard string, p if masterAlias == nil { results <- fmt.Errorf("no master for shard %v/%v", keyspace, shard) - } else if !topo.TabletAliasEqual(shardInfo.MasterAlias, masterAlias) { - results <- fmt.Errorf("master mismatch for shard %v/%v: found %v, expected %v", keyspace, shard, topo.TabletAliasString(masterAlias), topo.TabletAliasString(shardInfo.MasterAlias)) + } else if !topoproto.TabletAliasEqual(shardInfo.MasterAlias, masterAlias) { + results <- fmt.Errorf("master mismatch for shard %v/%v: found %v, expected %v", keyspace, shard, topoproto.TabletAliasString(masterAlias), topoproto.TabletAliasString(shardInfo.MasterAlias)) } for _, alias := range aliases { @@ -151,9 +152,9 @@ func (wr *Wrangler) validateShard(ctx context.Context, keyspace, shard string, p go func(alias *pb.TabletAlias) { defer wg.Done() if err := topo.Validate(ctx, wr.ts, alias); err != nil { - results <- fmt.Errorf("Validate(%v) failed: %v", topo.TabletAliasString(alias), err) + results <- fmt.Errorf("Validate(%v) failed: %v", topoproto.TabletAliasString(alias), err) } else { - wr.Logger().Infof("tablet %v is valid", topo.TabletAliasString(alias)) + wr.Logger().Infof("tablet %v is valid", topoproto.TabletAliasString(alias)) } }(alias) } @@ -179,7 +180,7 @@ func normalizeIP(ip string) string { func (wr *Wrangler) validateReplication(ctx context.Context, shardInfo *topo.ShardInfo, tabletMap map[pb.TabletAlias]*topo.TabletInfo, results chan<- error) { masterTablet, ok := tabletMap[*shardInfo.MasterAlias] if !ok { - results <- fmt.Errorf("master %v not in tablet map", topo.TabletAliasString(shardInfo.MasterAlias)) + results <- fmt.Errorf("master %v not in tablet map", topoproto.TabletAliasString(shardInfo.MasterAlias)) return } @@ -189,7 +190,7 @@ func (wr *Wrangler) validateReplication(ctx context.Context, shardInfo *topo.Sha return } if len(slaveList) == 0 { - results <- fmt.Errorf("no slaves of tablet %v found", topo.TabletAliasString(shardInfo.MasterAlias)) + results <- fmt.Errorf("no slaves of tablet %v found", topoproto.TabletAliasString(shardInfo.MasterAlias)) return } @@ -214,7 +215,7 @@ func (wr *Wrangler) validateReplication(ctx context.Context, shardInfo *topo.Sha } if !slaveIPMap[normalizeIP(tablet.Ip)] { - results <- fmt.Errorf("slave %v not replicating: %v slave list: %q", topo.TabletAliasString(tablet.Alias), tablet.Ip, slaveList) + results <- fmt.Errorf("slave %v not replicating: %v slave list: %q", topoproto.TabletAliasString(tablet.Alias), tablet.Ip, slaveList) } } } @@ -226,7 +227,7 @@ func (wr *Wrangler) pingTablets(ctx context.Context, tabletMap map[pb.TabletAlia defer wg.Done() if err := wr.tmc.Ping(ctx, tabletInfo); err != nil { - results <- fmt.Errorf("Ping(%v) failed: %v tablet hostname: %v", topo.TabletAliasString(&tabletAlias), err, tabletInfo.Hostname) + results <- fmt.Errorf("Ping(%v) failed: %v tablet hostname: %v", topoproto.TabletAliasString(&tabletAlias), err, tabletInfo.Hostname) } }(tabletAlias, tabletInfo) } diff --git a/go/vt/wrangler/version.go b/go/vt/wrangler/version.go index 043e4e2357..023e146d52 100644 --- a/go/vt/wrangler/version.go +++ b/go/vt/wrangler/version.go @@ -15,6 +15,7 @@ import ( log "github.com/golang/glog" "github.com/youtube/vitess/go/vt/concurrency" "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "golang.org/x/net/context" pb "github.com/youtube/vitess/go/vt/proto/topodata" @@ -67,14 +68,14 @@ func (wr *Wrangler) GetVersion(ctx context.Context, tabletAlias *pb.TabletAlias) if err != nil { return "", err } - log.Infof("Tablet %v is running version '%v'", topo.TabletAliasString(tabletAlias), version) + log.Infof("Tablet %v is running version '%v'", topoproto.TabletAliasString(tabletAlias), version) return version, err } // helper method to asynchronously get and diff a version func (wr *Wrangler) diffVersion(ctx context.Context, masterVersion string, masterAlias *pb.TabletAlias, alias *pb.TabletAlias, wg *sync.WaitGroup, er concurrency.ErrorRecorder) { defer wg.Done() - log.Infof("Gathering version for %v", topo.TabletAliasString(alias)) + log.Infof("Gathering version for %v", topoproto.TabletAliasString(alias)) slaveVersion, err := wr.GetVersion(ctx, alias) if err != nil { er.RecordError(err) @@ -82,7 +83,7 @@ func (wr *Wrangler) diffVersion(ctx context.Context, masterVersion string, maste } if masterVersion != slaveVersion { - er.RecordError(fmt.Errorf("Master %v version %v is different than slave %v version %v", topo.TabletAliasString(masterAlias), masterVersion, topo.TabletAliasString(alias), slaveVersion)) + er.RecordError(fmt.Errorf("Master %v version %v is different than slave %v version %v", topoproto.TabletAliasString(masterAlias), masterVersion, topoproto.TabletAliasString(alias), slaveVersion)) } } @@ -95,10 +96,10 @@ func (wr *Wrangler) ValidateVersionShard(ctx context.Context, keyspace, shard st } // get version from the master, or error - if topo.TabletAliasIsZero(si.MasterAlias) { + if !si.HasMaster() { return fmt.Errorf("No master in shard %v/%v", keyspace, shard) } - log.Infof("Gathering version for master %v", topo.TabletAliasString(si.MasterAlias)) + log.Infof("Gathering version for master %v", topoproto.TabletAliasString(si.MasterAlias)) masterVersion, err := wr.GetVersion(ctx, si.MasterAlias) if err != nil { return err @@ -115,7 +116,7 @@ func (wr *Wrangler) ValidateVersionShard(ctx context.Context, keyspace, shard st er := concurrency.AllErrorRecorder{} wg := sync.WaitGroup{} for _, alias := range aliases { - if topo.TabletAliasEqual(alias, si.MasterAlias) { + if topoproto.TabletAliasEqual(alias, si.MasterAlias) { continue } @@ -152,11 +153,11 @@ func (wr *Wrangler) ValidateVersionKeyspace(ctx context.Context, keyspace string if err != nil { return err } - if topo.TabletAliasIsZero(si.MasterAlias) { + if !si.HasMaster() { return fmt.Errorf("No master in shard %v/%v", keyspace, shards[0]) } referenceAlias := si.MasterAlias - log.Infof("Gathering version for reference master %v", topo.TabletAliasString(referenceAlias)) + log.Infof("Gathering version for reference master %v", topoproto.TabletAliasString(referenceAlias)) referenceVersion, err := wr.GetVersion(ctx, referenceAlias) if err != nil { return err @@ -173,7 +174,7 @@ func (wr *Wrangler) ValidateVersionKeyspace(ctx context.Context, keyspace string } for _, alias := range aliases { - if topo.TabletAliasEqual(alias, si.MasterAlias) { + if topoproto.TabletAliasEqual(alias, si.MasterAlias) { continue } diff --git a/go/vt/wrangler/zkns.go b/go/vt/wrangler/zkns.go index c77586bb12..c5ed4af87f 100644 --- a/go/vt/wrangler/zkns.go +++ b/go/vt/wrangler/zkns.go @@ -9,6 +9,7 @@ import ( log "github.com/golang/glog" "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "github.com/youtube/vitess/go/vt/zktopo" "github.com/youtube/vitess/go/zk" "github.com/youtube/vitess/go/zk/zkns" @@ -154,7 +155,7 @@ func (wr *Wrangler) exportVtnsToZkns(ctx context.Context, zconn zk.Conn, vtnsAdd if tabletTypeStr == "action" || tabletTypeStr == "actionlog" { return nil, nil } - tabletType, err := topo.ParseTabletType(tabletTypeStr) + tabletType, err := topoproto.ParseTabletType(tabletTypeStr) if err != nil { return nil, err } diff --git a/go/vt/zktopo/serving_graph.go b/go/vt/zktopo/serving_graph.go index 9c2e1d516f..871626852b 100644 --- a/go/vt/zktopo/serving_graph.go +++ b/go/vt/zktopo/serving_graph.go @@ -14,6 +14,7 @@ import ( log "github.com/golang/glog" "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "github.com/youtube/vitess/go/zk" "golang.org/x/net/context" "launchpad.net/gozk/zookeeper" @@ -62,7 +63,7 @@ func (zkts *Server) GetSrvTabletTypesPerShard(ctx context.Context, cell, keyspac if tt == "action" || tt == "actionlog" { continue } - if ptt, err := topo.ParseTabletType(tt); err == nil { + if ptt, err := topoproto.ParseTabletType(tt); err == nil { result = append(result, ptt) } } diff --git a/go/vt/zktopo/tablet.go b/go/vt/zktopo/tablet.go index 35fee551a3..50c1199cd0 100644 --- a/go/vt/zktopo/tablet.go +++ b/go/vt/zktopo/tablet.go @@ -12,6 +12,7 @@ import ( "github.com/youtube/vitess/go/event" "github.com/youtube/vitess/go/vt/topo" "github.com/youtube/vitess/go/vt/topo/events" + "github.com/youtube/vitess/go/vt/topo/topoproto" "github.com/youtube/vitess/go/zk" "golang.org/x/net/context" "launchpad.net/gozk/zookeeper" @@ -25,7 +26,7 @@ This file contains the tablet management parts of zktopo.Server // TabletPathForAlias converts a tablet alias to the zk path func TabletPathForAlias(alias *pb.TabletAlias) string { - return fmt.Sprintf("/zk/%v/vt/tablets/%v", alias.Cell, topo.TabletAliasUIDStr(alias)) + return fmt.Sprintf("/zk/%v/vt/tablets/%v", alias.Cell, topoproto.TabletAliasUIDStr(alias)) } func tabletDirectoryForCell(cell string) string { @@ -200,7 +201,7 @@ func (zkts *Server) GetTabletsByCell(ctx context.Context, cell string) ([]*pb.Ta sort.Strings(children) result := make([]*pb.TabletAlias, len(children)) for i, child := range children { - uid, err := topo.ParseUID(child) + uid, err := topoproto.ParseUID(child) if err != nil { return nil, err } From 4fb689e8e92b91564204c5816825ef9d258752c9 Mon Sep 17 00:00:00 2001 From: Michael Berlin Date: Wed, 19 Aug 2015 14:38:58 -0700 Subject: [PATCH 57/80] vtctl: WaitForFilteredReplication: Made allowedHealthErrorsInARow a flag. --- go/vt/vtctl/vtctl.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/go/vt/vtctl/vtctl.go b/go/vt/vtctl/vtctl.go index 7f8bdc7e32..6097d01a3f 100644 --- a/go/vt/vtctl/vtctl.go +++ b/go/vt/vtctl/vtctl.go @@ -1339,6 +1339,12 @@ func commandWaitForFilteredReplication(ctx context.Context, wr *wrangler.Wrangle "Specifies the maximum delay, in seconds, the filtered replication of the"+ " given destination shard should lag behind the source shard. When"+ " higher, the command will block and wait for the delay to decrease.") + // In case of automated reshardings, a tablet may still report itself as + // unhealthy e.g. because CopySchemaShard wasn't run yet and the db doesn't + // exist yet. + allowedHealthErrorsInARow := subFlags.Int("allowed_health_errors_in_a_row", 3, + "Limit of observed health errors in a row after which the command will fail and no longer wait for the tablet to become healthy.") + if err := subFlags.Parse(args); err != nil { return err } @@ -1380,11 +1386,7 @@ func commandWaitForFilteredReplication(ctx context.Context, wr *wrangler.Wrangle return fmt.Errorf("could not stream health records from tablet: %v err: %v", alias, err) } var lastSeenDelay int - healthErrorInARow := 0 - // In case of automated reshardings, a tablet may still report itself as - // unhealthy e.g. because CopySchemaShard wasn't run yet and the db doesn't - // exist yet. - const allowedHealthErrorInARow = 3 + healthErrorsInARow := 0 for { select { case <-ctx.Done(): @@ -1398,17 +1400,17 @@ func commandWaitForFilteredReplication(ctx context.Context, wr *wrangler.Wrangle return fmt.Errorf("health record does not include RealtimeStats message. tablet: %v health record: %v", alias, shr) } if stats.HealthError != "" { - healthErrorInARow++ - if healthErrorInARow >= allowedHealthErrorInARow { + healthErrorsInARow++ + if healthErrorsInARow >= *allowedHealthErrorsInARow { return fmt.Errorf("tablet is not healthy. tablet: %v health record: %v", alias, shr) } wr.Logger().Printf("Tablet is not healthy. Waiting for %v more health"+ " record(s) before the command will fail."+ " tablet: %v health record: %v\n", - (allowedHealthErrorInARow - healthErrorInARow), alias, shr) + (*allowedHealthErrorsInARow - healthErrorsInARow), alias, shr) continue } else { - healthErrorInARow = 0 + healthErrorsInARow = 0 } if stats.BinlogPlayersCount == 0 { From 1ee4476925baf4e3b7f53739e5eb15816ab0d3ca Mon Sep 17 00:00:00 2001 From: Ammar Aijazi Date: Wed, 19 Aug 2015 15:39:06 -0700 Subject: [PATCH 58/80] Demote MySQL deadlocks to INFO logs --- go/vt/tabletserver/tablet_error.go | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/go/vt/tabletserver/tablet_error.go b/go/vt/tabletserver/tablet_error.go index c6ea2c097f..48619dd4d4 100644 --- a/go/vt/tabletserver/tablet_error.go +++ b/go/vt/tabletserver/tablet_error.go @@ -193,13 +193,20 @@ func handleError(err *error, logStats *SQLQueryStats, queryServiceStats *QuerySe } *err = terr terr.RecordStats(queryServiceStats) - if terr.ErrorType == ErrRetry { // Retry errors are too spammy + switch terr.ErrorType { + case ErrRetry: // Retry errors are too spammy return - } - if terr.ErrorType == ErrTxPoolFull { + case ErrTxPoolFull: logTxPoolFull.Errorf("%v", terr) - } else { - log.Errorf("%v", terr) + default: + switch terr.SqlError { + // MySQL deadlock errors are (usually) due to client behavior, not server + // behavior, and therefore logged at the INFO level. + case mysql.ErrLockWaitTimeout, mysql.ErrLockDeadlock: + log.Infof("%v", terr) + default: + log.Errorf("%v", terr) + } } } if logStats != nil { From 0dffe50a724f3727b2a5b13a53dea4f4daac4563 Mon Sep 17 00:00:00 2001 From: Alain Jobart Date: Wed, 19 Aug 2015 15:48:38 -0700 Subject: [PATCH 59/80] Splitting topo.Server into topo.Impl (the interface for underlying implementations), and topo.Server (the container with extra methods clients should use). Not converting all methods yet. --- go/cmd/vtctld/api.go | 10 ++---- go/cmd/vtctld/plugin_etcdtopo.go | 2 +- go/cmd/vtctld/plugin_zktopo.go | 2 +- go/cmd/vtctld/vtctld.go | 8 ++--- go/cmd/vtgate/vtgate.go | 7 +--- go/vt/etcdtopo/explorer_test.go | 8 ++--- go/vt/etcdtopo/server_test.go | 7 ++-- go/vt/janitor/janitor_test.go | 3 +- go/vt/schemamanager/schemamanager_test.go | 6 ++-- go/vt/tabletmanager/after_action.go | 2 +- go/vt/tabletmanager/reparent.go | 2 +- go/vt/topo/helpers/tee.go | 35 +++++++++++++++----- go/vt/topo/helpers/tee_test.go | 2 +- go/vt/topo/helpers/tee_topo_test.go | 2 +- go/vt/topo/server.go | 40 ++++++++++++++++------- go/vt/topo/shard.go | 8 ++--- go/vt/topo/test/faketopo/faketopo.go | 10 ++++++ go/vt/topo/test/shard.go | 8 ++--- go/vt/topo/test/vschema.go | 17 ++++------ go/vt/vtctl/plugin_zktopo.go | 4 +-- go/vt/vtctl/vtctl.go | 14 ++------ go/vt/vtgate/srv_topo_server_test.go | 6 ++-- go/vt/worker/clone_utils.go | 2 +- go/vt/wrangler/zkns.go | 4 +-- go/vt/zktopo/server.go | 6 ++-- go/vt/zktopo/testserver.go | 22 +++---------- go/vt/zktopo/testserver_test.go | 2 +- go/vt/zktopo/zktopo_test.go | 4 +-- 28 files changed, 126 insertions(+), 117 deletions(-) diff --git a/go/cmd/vtctld/api.go b/go/cmd/vtctld/api.go index 6def409ced..cc43bfaa8d 100644 --- a/go/cmd/vtctld/api.go +++ b/go/cmd/vtctld/api.go @@ -248,12 +248,6 @@ func initAPI(ctx context.Context, ts topo.Server, actions *ActionRepository) { // VSchema http.HandleFunc(apiPrefix+"vschema/", func(w http.ResponseWriter, r *http.Request) { - schemafier, ok := ts.(topo.Schemafier) - if !ok { - httpErrorf(w, r, "%T doesn't support schemafier API", ts) - return - } - // Save VSchema if r.Method == "POST" { vschema, err := ioutil.ReadAll(r.Body) @@ -261,14 +255,14 @@ func initAPI(ctx context.Context, ts topo.Server, actions *ActionRepository) { httpErrorf(w, r, "can't read request body: %v", err) return } - if err := schemafier.SaveVSchema(ctx, string(vschema)); err != nil { + if err := ts.SaveVSchema(ctx, string(vschema)); err != nil { httpErrorf(w, r, "can't save vschema: %v", err) } return } // Get VSchema - vschema, err := schemafier.GetVSchema(ctx) + vschema, err := ts.GetVSchema(ctx) if err != nil { httpErrorf(w, r, "can't get vschema: %v", err) return diff --git a/go/cmd/vtctld/plugin_etcdtopo.go b/go/cmd/vtctld/plugin_etcdtopo.go index 0dfba08ed8..f85a4d7185 100644 --- a/go/cmd/vtctld/plugin_etcdtopo.go +++ b/go/cmd/vtctld/plugin_etcdtopo.go @@ -15,7 +15,7 @@ import ( func init() { // Wait until flags are parsed, so we can check which topo server is in use. servenv.OnRun(func() { - if etcdServer, ok := topo.GetServer().(*etcdtopo.Server); ok { + if etcdServer, ok := topo.GetServer().Impl.(*etcdtopo.Server); ok { HandleExplorer("etcd", "/etcd/", "etcd.html", etcdtopo.NewExplorer(etcdServer)) } }) diff --git a/go/cmd/vtctld/plugin_zktopo.go b/go/cmd/vtctld/plugin_zktopo.go index 1b7fa3d223..a5e17cf4a6 100644 --- a/go/cmd/vtctld/plugin_zktopo.go +++ b/go/cmd/vtctld/plugin_zktopo.go @@ -29,7 +29,7 @@ import ( func init() { // Wait until flags are parsed, so we can check which topo server is in use. servenv.OnRun(func() { - if zkServer, ok := topo.GetServer().(*zktopo.Server); ok { + if zkServer, ok := topo.GetServer().Impl.(*zktopo.Server); ok { HandleExplorer("zk", "/zk/", "zk.html", NewZkExplorer(zkServer.GetZConn())) } }) diff --git a/go/cmd/vtctld/vtctld.go b/go/cmd/vtctld/vtctld.go index d771479059..f9f5e70509 100644 --- a/go/cmd/vtctld/vtctld.go +++ b/go/cmd/vtctld/vtctld.go @@ -328,10 +328,6 @@ func main() { httpErrorf(w, r, "cannot parse form: %s", err) return } - schemafier, ok := ts.(topo.Schemafier) - if !ok { - httpErrorf(w, r, "%s", fmt.Errorf("%T doesn't support schemafier API", ts)) - } var data struct { Error error Input, Output string @@ -340,9 +336,9 @@ func main() { switch r.Method { case "POST": data.Input = r.FormValue("vschema") - data.Error = schemafier.SaveVSchema(ctx, data.Input) + data.Error = ts.SaveVSchema(ctx, data.Input) } - vschema, err := schemafier.GetVSchema(ctx) + vschema, err := ts.GetVSchema(ctx) if err != nil { if data.Error == nil { data.Error = fmt.Errorf("Error fetching schema: %s", err) diff --git a/go/cmd/vtgate/vtgate.go b/go/cmd/vtgate/vtgate.go index 4c120d1c25..ab282e08a2 100644 --- a/go/cmd/vtgate/vtgate.go +++ b/go/cmd/vtgate/vtgate.go @@ -55,13 +55,8 @@ func main() { } log.Infof("v3 is enabled: loaded schema from file: %v", *schemaFile) } else { - schemafier, ok := ts.(topo.Schemafier) - if !ok { - log.Infof("Skipping v3 initialization: topo does not suppurt schemafier interface") - goto startServer - } ctx := context.Background() - schemaJSON, err := schemafier.GetVSchema(ctx) + schemaJSON, err := ts.GetVSchema(ctx) if err != nil { log.Warningf("Skipping v3 initialization: GetVSchema failed: %v", err) goto startServer diff --git a/go/vt/etcdtopo/explorer_test.go b/go/vt/etcdtopo/explorer_test.go index fa0b2b2fd4..0202a567a3 100644 --- a/go/vt/etcdtopo/explorer_test.go +++ b/go/vt/etcdtopo/explorer_test.go @@ -73,7 +73,7 @@ func TestHandlePathRoot(t *testing.T) { want := []string{"global", "cell1", "cell2", "cell3"} ts := newTestServer(t, cells) - ex := NewExplorer(ts) + ex := NewExplorer(ts.Impl.(*Server)) result := ex.HandlePath(nil, input, nil) exResult := result.(*explorerResult) if got := exResult.Children; !reflect.DeepEqual(got, want) { @@ -101,7 +101,7 @@ func TestHandlePathKeyspace(t *testing.T) { } m := &mockActionRepo{} - ex := NewExplorer(ts) + ex := NewExplorer(ts.Impl.(*Server)) result := ex.HandlePath(m, input, nil) exResult := result.(*explorerResult) if got := exResult.Data; got != want { @@ -135,7 +135,7 @@ func TestHandlePathShard(t *testing.T) { } m := &mockActionRepo{} - ex := NewExplorer(ts) + ex := NewExplorer(ts.Impl.(*Server)) result := ex.HandlePath(m, input, nil) exResult := result.(*explorerResult) if got := exResult.Data; got != want { @@ -169,7 +169,7 @@ func TestHandlePathTablet(t *testing.T) { } m := &mockActionRepo{} - ex := NewExplorer(ts) + ex := NewExplorer(ts.Impl.(*Server)) result := ex.HandlePath(m, input, nil) exResult := result.(*explorerResult) if got := exResult.Data; got != want { diff --git a/go/vt/etcdtopo/server_test.go b/go/vt/etcdtopo/server_test.go index 0075bc6b81..ad39f5d240 100644 --- a/go/vt/etcdtopo/server_test.go +++ b/go/vt/etcdtopo/server_test.go @@ -8,11 +8,12 @@ import ( "testing" "github.com/youtube/vitess/go/flagutil" + "github.com/youtube/vitess/go/vt/topo" "github.com/youtube/vitess/go/vt/topo/test" "golang.org/x/net/context" ) -func newTestServer(t *testing.T, cells []string) *Server { +func newTestServer(t *testing.T, cells []string) topo.Server { s := &Server{ _cells: make(map[string]*cellClient), newClient: newTestClient, @@ -27,7 +28,9 @@ func newTestServer(t *testing.T, cells []string) *Server { c.Set("/vt/cells/"+cell, cell, 0) } - return s + return topo.Server{ + Impl: s, + } } func TestKeyspace(t *testing.T) { diff --git a/go/vt/janitor/janitor_test.go b/go/vt/janitor/janitor_test.go index 75e6caa13b..724bb6ad47 100644 --- a/go/vt/janitor/janitor_test.go +++ b/go/vt/janitor/janitor_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + "github.com/youtube/vitess/go/vt/topo" "github.com/youtube/vitess/go/vt/wrangler" ) @@ -49,7 +50,7 @@ func TestJanitorInfo(t *testing.T) { } func TestRunJanitor(t *testing.T) { - scheduler, _ := New("a", "a", nil, nil, 0) + scheduler, _ := New("a", "a", topo.Server{}, nil, 0) jan := newTestJanitor() ji := newJanitorInfo(jan) scheduler.janitors["test"] = ji diff --git a/go/vt/schemamanager/schemamanager_test.go b/go/vt/schemamanager/schemamanager_test.go index 4dac2ae44b..7aa18818b9 100644 --- a/go/vt/schemamanager/schemamanager_test.go +++ b/go/vt/schemamanager/schemamanager_test.go @@ -271,8 +271,10 @@ type fakeTopo struct { faketopo.FakeTopo } -func newFakeTopo() *fakeTopo { - return &fakeTopo{} +func newFakeTopo() topo.Server { + return topo.Server{ + Impl: &fakeTopo{}, + } } func (topoServer *fakeTopo) GetShardNames(ctx context.Context, keyspace string) ([]string, error) { diff --git a/go/vt/tabletmanager/after_action.go b/go/vt/tabletmanager/after_action.go index 236501de7c..25cf7352e7 100644 --- a/go/vt/tabletmanager/after_action.go +++ b/go/vt/tabletmanager/after_action.go @@ -161,7 +161,7 @@ func (agent *ActionAgent) changeCallback(ctx context.Context, oldTablet, newTabl var blacklistedTables []string var err error if allowQuery { - shardInfo, err = topo.GetShard(ctx, agent.TopoServer, newTablet.Keyspace, newTablet.Shard) + shardInfo, err = agent.TopoServer.GetShard(ctx, newTablet.Keyspace, newTablet.Shard) if err != nil { log.Errorf("Cannot read shard for this tablet %v, might have inaccurate SourceShards and TabletControls: %v", newTablet.Alias, err) } else { diff --git a/go/vt/tabletmanager/reparent.go b/go/vt/tabletmanager/reparent.go index 110e684f46..7ac9cc9332 100644 --- a/go/vt/tabletmanager/reparent.go +++ b/go/vt/tabletmanager/reparent.go @@ -56,7 +56,7 @@ func (agent *ActionAgent) TabletExternallyReparented(ctx context.Context, extern tablet := agent.Tablet() // Check the global shard record. - si, err := topo.GetShard(ctx, agent.TopoServer, tablet.Keyspace, tablet.Shard) + si, err := agent.TopoServer.GetShard(ctx, tablet.Keyspace, tablet.Shard) if err != nil { log.Warningf("fastTabletExternallyReparented: failed to read global shard record for %v/%v: %v", tablet.Keyspace, tablet.Shard, err) return err diff --git a/go/vt/topo/helpers/tee.go b/go/vt/topo/helpers/tee.go index b73d1d7ff2..b406a4ecb8 100644 --- a/go/vt/topo/helpers/tee.go +++ b/go/vt/topo/helpers/tee.go @@ -26,14 +26,14 @@ import ( // - we lock primary/secondary if reverseLockOrder is False, // or secondary/primary if reverseLockOrder is True. type Tee struct { - primary topo.Server - secondary topo.Server + primary topo.Impl + secondary topo.Impl - readFrom topo.Server - readFromSecond topo.Server + readFrom topo.Impl + readFromSecond topo.Impl - lockFirst topo.Server - lockSecond topo.Server + lockFirst topo.Impl + lockSecond topo.Impl // protects the variables below this point mu sync.Mutex @@ -55,8 +55,8 @@ type versionMapping struct { readFromSecondVersion int64 } -// NewTee returns a new topo.Server object -func NewTee(primary, secondary topo.Server, reverseLockOrder bool) *Tee { +// NewTee returns a new topo.Impl object +func NewTee(primary, secondary topo.Impl, reverseLockOrder bool) *Tee { lockFirst := primary lockSecond := secondary if reverseLockOrder { @@ -799,3 +799,22 @@ func (tee *Tee) UnlockShardForAction(ctx context.Context, keyspace, shard, lockP } return perr } + +// SaveVSchema is part of the topo.Server interface +func (tee *Tee) SaveVSchema(ctx context.Context, contents string) error { + err := tee.primary.SaveVSchema(ctx, contents) + if err != nil { + return err + } + + if err := tee.secondary.SaveVSchema(ctx, contents); err != nil { + // not critical enough to fail + log.Warningf("secondary.SaveVSchema() failed: %v", err) + } + return err +} + +// GetVSchema is part of the topo.Server interface +func (tee *Tee) GetVSchema(ctx context.Context) (string, error) { + return tee.readFrom.GetVSchema(ctx) +} diff --git a/go/vt/topo/helpers/tee_test.go b/go/vt/topo/helpers/tee_test.go index 0cf8144324..cc4962a765 100644 --- a/go/vt/topo/helpers/tee_test.go +++ b/go/vt/topo/helpers/tee_test.go @@ -25,7 +25,7 @@ func TestTee(t *testing.T) { // create a tee and check it implements the interface tee := NewTee(fromTS, toTS, true) - var _ topo.Server = tee + var _ topo.Impl = tee // create a keyspace, make sure it is on both sides if err := tee.CreateKeyspace(ctx, "keyspace2", &pb.Keyspace{}); err != nil { diff --git a/go/vt/topo/helpers/tee_topo_test.go b/go/vt/topo/helpers/tee_topo_test.go index 1650110c39..57573fefff 100644 --- a/go/vt/topo/helpers/tee_topo_test.go +++ b/go/vt/topo/helpers/tee_topo_test.go @@ -45,7 +45,7 @@ func newFakeTeeServer(t *testing.T) topo.Server { s1 := fakeServer{Server: zktopo.NewServer(zconn1), localCells: cells[:len(cells)-1]} s2 := fakeServer{Server: zktopo.NewServer(zconn2), localCells: cells[:len(cells)-1]} - return NewTee(s1, s2, false) + return topo.Server{Impl: NewTee(s1, s2, false)} } func TestKeyspace(t *testing.T) { diff --git a/go/vt/topo/server.go b/go/vt/topo/server.go index 31022f5a58..c98cc94db9 100644 --- a/go/vt/topo/server.go +++ b/go/vt/topo/server.go @@ -45,14 +45,14 @@ var ( ErrPartialResult = errors.New("partial result") ) -// Server is the interface used to talk to a persistent +// Impl is the interface used to talk to a persistent // backend storage server and locking service. // // Zookeeper is a good example of this, and zktopo contains the // implementation for this using zookeeper. // // Inside Google, we use Chubby. -type Server interface { +type Impl interface { // topo.Server management interface. Close() @@ -305,17 +305,35 @@ type Server interface { // UnlockShardForAction unlocks a shard. UnlockShardForAction(ctx context.Context, keyspace, shard, lockPath, results string) error + + // + // V3 Schema management, global + // + + // SaveVSchema saves the provided schema in the topo server. + SaveVSchema(context.Context, string) error + + // GetVSchema retrieves the schema from the topo server. + // + // If no schema has been previously saved, it should return "{}" + GetVSchema(ctx context.Context) (string, error) +} + +// Server is a wrapper type that can have extra methods. +// Outside modules should just use the Server object. +type Server struct { + Impl } // Schemafier is a temporary interface for supporting vschema // reads and writes. It will eventually be merged into Server. -type Schemafier interface { - SaveVSchema(context.Context, string) error - GetVSchema(ctx context.Context) (string, error) -} +//type Schemafier interface { +// SaveVSchema(context.Context, string) error +// GetVSchema(ctx context.Context) (string, error) +//} // Registry for Server implementations. -var serverImpls = make(map[string]Server) +var serverImpls = make(map[string]Impl) // Which implementation to use var topoImplementation = flag.String("topo_implementation", "zookeeper", "the topology implementation to use") @@ -323,7 +341,7 @@ var topoImplementation = flag.String("topo_implementation", "zookeeper", "the to // RegisterServer adds an implementation for a Server. // If an implementation with that name already exists, panics. // Call this in the 'init' function in your module. -func RegisterServer(name string, ts Server) { +func RegisterServer(name string, ts Impl) { if serverImpls[name] != nil { panic(fmt.Errorf("Duplicate topo.Server registration for %v", name)) } @@ -332,7 +350,7 @@ func RegisterServer(name string, ts Server) { // GetServerByName returns a specific Server by name, or nil. func GetServerByName(name string) Server { - return serverImpls[name] + return Server{Impl: serverImpls[name]} } // GetServer returns 'our' Server, going down this list: @@ -344,7 +362,7 @@ func GetServer() Server { if len(serverImpls) == 1 { for name, ts := range serverImpls { log.V(6).Infof("Using only topo.Server: %v", name) - return ts + return Server{Impl: ts} } } @@ -353,7 +371,7 @@ func GetServer() Server { panic(fmt.Errorf("No topo.Server named %v", *topoImplementation)) } log.V(6).Infof("Using topo.Server: %v", *topoImplementation) - return result + return Server{Impl: result} } // CloseServers closes all registered Server. diff --git a/go/vt/topo/shard.go b/go/vt/topo/shard.go index 231e0e320e..25916cc149 100644 --- a/go/vt/topo/shard.go +++ b/go/vt/topo/shard.go @@ -191,14 +191,14 @@ func (si *ShardInfo) HasCell(cell string) bool { // GetShard is a high level function to read shard data. // It generates trace spans. -func GetShard(ctx context.Context, ts Server, keyspace, shard string) (*ShardInfo, error) { +func (ts Server) GetShard(ctx context.Context, keyspace, shard string) (*ShardInfo, error) { span := trace.NewSpanFromContext(ctx) span.StartClient("TopoServer.GetShard") span.Annotate("keyspace", keyspace) span.Annotate("shard", shard) defer span.Finish() - return ts.GetShard(ctx, keyspace, shard) + return ts.Impl.GetShard(ctx, keyspace, shard) } // UpdateShard updates the shard data, with the right version @@ -227,7 +227,7 @@ func UpdateShard(ctx context.Context, ts Server, si *ShardInfo) error { // If the update succeeds, it returns the updated ShardInfo. func UpdateShardFields(ctx context.Context, ts Server, keyspace, shard string, update func(*pb.Shard) error) (*ShardInfo, error) { for { - si, err := GetShard(ctx, ts, keyspace, shard) + si, err := ts.GetShard(ctx, keyspace, shard) if err != nil { return nil, err } @@ -528,7 +528,7 @@ func FindAllTabletAliasesInShardByCell(ctx context.Context, ts Server, keyspace, ctx = trace.NewContext(ctx, span) // read the shard information to find the cells - si, err := GetShard(ctx, ts, keyspace, shard) + si, err := ts.GetShard(ctx, keyspace, shard) if err != nil { return nil, err } diff --git a/go/vt/topo/test/faketopo/faketopo.go b/go/vt/topo/test/faketopo/faketopo.go index 5710d1b8ef..0493a8a8d5 100644 --- a/go/vt/topo/test/faketopo/faketopo.go +++ b/go/vt/topo/test/faketopo/faketopo.go @@ -228,3 +228,13 @@ func (ft FakeTopo) LockShardForAction(ctx context.Context, keyspace, shard, cont func (ft FakeTopo) UnlockShardForAction(ctx context.Context, keyspace, shard, lockPath, results string) error { return errNotImplemented } + +// SaveVSchema implements topo.Server. +func (ft FakeTopo) SaveVSchema(context.Context, string) error { + return errNotImplemented +} + +// GetVSchema implements topo.Server. +func (ft FakeTopo) GetVSchema(ctx context.Context) (string, error) { + return "", errNotImplemented +} diff --git a/go/vt/topo/test/shard.go b/go/vt/topo/test/shard.go index 5b2ab7de3c..e6331d9fec 100644 --- a/go/vt/topo/test/shard.go +++ b/go/vt/topo/test/shard.go @@ -57,11 +57,11 @@ func CheckShard(ctx context.Context, t *testing.T, ts topo.Server) { t.Fatalf("CreateShard: %v", err) } - if _, err := topo.GetShard(ctx, ts, "test_keyspace", "666"); err != topo.ErrNoNode { + if _, err := ts.GetShard(ctx, "test_keyspace", "666"); err != topo.ErrNoNode { t.Errorf("GetShard(666): %v", err) } - shardInfo, err := topo.GetShard(ctx, ts, "test_keyspace", "b0-c0") + shardInfo, err := ts.GetShard(ctx, "test_keyspace", "b0-c0") if err != nil { t.Errorf("GetShard: %v", err) } @@ -115,7 +115,7 @@ func CheckShard(ctx context.Context, t *testing.T, ts topo.Server) { if err != nil { t.Fatalf("UpdateShardFields error: %v", err) } - si, err := topo.GetShard(ctx, ts, "test_keyspace", "b0-c0") + si, err := ts.GetShard(ctx, "test_keyspace", "b0-c0") if err != nil { t.Fatalf("GetShard: %v", err) } @@ -130,7 +130,7 @@ func CheckShard(ctx context.Context, t *testing.T, ts topo.Server) { t.Fatalf("UpdateShardFields error: %v", err) } - updatedShardInfo, err := topo.GetShard(ctx, ts, "test_keyspace", "b0-c0") + updatedShardInfo, err := ts.GetShard(ctx, "test_keyspace", "b0-c0") if err != nil { t.Fatalf("GetShard: %v", err) } diff --git a/go/vt/topo/test/vschema.go b/go/vt/topo/test/vschema.go index 0b66d847d0..3405bc376d 100644 --- a/go/vt/topo/test/vschema.go +++ b/go/vt/topo/test/vschema.go @@ -14,12 +14,7 @@ import ( // CheckVSchema runs the tests on the VSchema part of the API func CheckVSchema(ctx context.Context, t *testing.T, ts topo.Server) { - schemafier, ok := ts.(topo.Schemafier) - if !ok { - t.Errorf("%T is not a Schemafier", ts) - return - } - got, err := schemafier.GetVSchema(ctx) + got, err := ts.GetVSchema(ctx) if err != nil { t.Error(err) } @@ -28,12 +23,12 @@ func CheckVSchema(ctx context.Context, t *testing.T, ts topo.Server) { t.Errorf("GetVSchema: %s, want %s", got, want) } - err = schemafier.SaveVSchema(ctx, `{ "Keyspaces": {}}`) + err = ts.SaveVSchema(ctx, `{ "Keyspaces": {}}`) if err != nil { t.Error(err) } - got, err = schemafier.GetVSchema(ctx) + got, err = ts.GetVSchema(ctx) if err != nil { t.Error(err) } @@ -42,12 +37,12 @@ func CheckVSchema(ctx context.Context, t *testing.T, ts topo.Server) { t.Errorf("GetVSchema: %s, want %s", got, want) } - err = schemafier.SaveVSchema(ctx, `{ "Keyspaces": { "aa": { "Sharded": false}}}`) + err = ts.SaveVSchema(ctx, `{ "Keyspaces": { "aa": { "Sharded": false}}}`) if err != nil { t.Error(err) } - got, err = schemafier.GetVSchema(ctx) + got, err = ts.GetVSchema(ctx) if err != nil { t.Error(err) } @@ -56,7 +51,7 @@ func CheckVSchema(ctx context.Context, t *testing.T, ts topo.Server) { t.Errorf("GetVSchema: %s, want %s", got, want) } - err = schemafier.SaveVSchema(ctx, "invalid") + err = ts.SaveVSchema(ctx, "invalid") want = "Unmarshal failed:" if err == nil || !strings.HasPrefix(err.Error(), want) { t.Errorf("SaveVSchema: %v, must start with %s", err, want) diff --git a/go/vt/vtctl/plugin_zktopo.go b/go/vt/vtctl/plugin_zktopo.go index f114cfdd26..ce9de86e0c 100644 --- a/go/vt/vtctl/plugin_zktopo.go +++ b/go/vt/vtctl/plugin_zktopo.go @@ -43,7 +43,7 @@ func init() { } func zkResolveWildcards(wr *wrangler.Wrangler, args []string) ([]string, error) { - zkts, ok := wr.TopoServer().(*zktopo.Server) + zkts, ok := wr.TopoServer().Impl.(*zktopo.Server) if !ok { return args, nil } @@ -65,7 +65,7 @@ func commandPruneActionLogs(ctx context.Context, wr *wrangler.Wrangler, subFlags return err } - zkts, ok := wr.TopoServer().(*zktopo.Server) + zkts, ok := wr.TopoServer().Impl.(*zktopo.Server) if !ok { return fmt.Errorf("PruneActionLogs requires a zktopo.Server") } diff --git a/go/vt/vtctl/vtctl.go b/go/vt/vtctl/vtctl.go index 8ba7d2a229..d8877aa236 100644 --- a/go/vt/vtctl/vtctl.go +++ b/go/vt/vtctl/vtctl.go @@ -2027,12 +2027,7 @@ func commandGetVSchema(ctx context.Context, wr *wrangler.Wrangler, subFlags *fla if subFlags.NArg() != 0 { return fmt.Errorf("The GetVSchema command does not support any arguments.") } - ts := wr.TopoServer() - schemafier, ok := ts.(topo.Schemafier) - if !ok { - return fmt.Errorf("%T does not support the vschema operations", ts) - } - schema, err := schemafier.GetVSchema(ctx) + schema, err := wr.TopoServer().GetVSchema(ctx) if err != nil { return err } @@ -2049,11 +2044,6 @@ func commandApplyVSchema(ctx context.Context, wr *wrangler.Wrangler, subFlags *f if (*vschema == "") == (*vschemaFile == "") { return fmt.Errorf("Either the vschema or vschemaFile flag must be specified when calling the ApplyVSchema command.") } - ts := wr.TopoServer() - schemafier, ok := ts.(topo.Schemafier) - if !ok { - return fmt.Errorf("%T does not support vschema operations", ts) - } s := *vschema if *vschemaFile != "" { schema, err := ioutil.ReadFile(*vschemaFile) @@ -2062,7 +2052,7 @@ func commandApplyVSchema(ctx context.Context, wr *wrangler.Wrangler, subFlags *f } s = string(schema) } - return schemafier.SaveVSchema(ctx, s) + return wr.TopoServer().SaveVSchema(ctx, s) } func commandGetSrvKeyspace(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { diff --git a/go/vt/vtgate/srv_topo_server_test.go b/go/vt/vtgate/srv_topo_server_test.go index b2e8177926..846cd3d8e5 100644 --- a/go/vt/vtgate/srv_topo_server_test.go +++ b/go/vt/vtgate/srv_topo_server_test.go @@ -253,7 +253,7 @@ func (ft *fakeTopoRemoteMaster) GetEndPoints(ctx context.Context, cell, keyspace // TestRemoteMaster will test getting endpoints for remote master. func TestRemoteMaster(t *testing.T) { ft := &fakeTopoRemoteMaster{cell: "cell1", remoteCell: "cell2"} - rsts := NewResilientSrvTopoServer(ft, "TestRemoteMaster") + rsts := NewResilientSrvTopoServer(topo.Server{Impl: ft}, "TestRemoteMaster") rsts.enableRemoteMaster = true // remote cell for master @@ -295,7 +295,7 @@ func TestRemoteMaster(t *testing.T) { // TestCacheWithErrors will test we properly return cached errors. func TestCacheWithErrors(t *testing.T) { ft := &fakeTopo{keyspace: "test_ks"} - rsts := NewResilientSrvTopoServer(ft, "TestCacheWithErrors") + rsts := NewResilientSrvTopoServer(topo.Server{Impl: ft}, "TestCacheWithErrors") // ask for the known keyspace, that populates the cache _, err := rsts.GetSrvKeyspace(context.Background(), "", "test_ks") @@ -322,7 +322,7 @@ func TestCacheWithErrors(t *testing.T) { // TestCachedErrors will test we properly return cached errors. func TestCachedErrors(t *testing.T) { ft := &fakeTopo{keyspace: "test_ks"} - rsts := NewResilientSrvTopoServer(ft, "TestCachedErrors") + rsts := NewResilientSrvTopoServer(topo.Server{Impl: ft}, "TestCachedErrors") // ask for an unknown keyspace, should get an error _, err := rsts.GetSrvKeyspace(context.Background(), "", "unknown_ks") diff --git a/go/vt/worker/clone_utils.go b/go/vt/worker/clone_utils.go index bab36a4306..658688888a 100644 --- a/go/vt/worker/clone_utils.go +++ b/go/vt/worker/clone_utils.go @@ -34,7 +34,7 @@ import ( func resolveDestinationShardMaster(ctx context.Context, keyspace, shard string, wr *wrangler.Wrangler) (*topo.TabletInfo, error) { var ti *topo.TabletInfo shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout) - si, err := topo.GetShard(shortCtx, wr.TopoServer(), keyspace, shard) + si, err := wr.TopoServer().GetShard(shortCtx, keyspace, shard) cancel() if err != nil { return ti, fmt.Errorf("unable to resolve destination shard %v/%v", keyspace, shard) diff --git a/go/vt/wrangler/zkns.go b/go/vt/wrangler/zkns.go index c5ed4af87f..9ad5618975 100644 --- a/go/vt/wrangler/zkns.go +++ b/go/vt/wrangler/zkns.go @@ -20,7 +20,7 @@ import ( // ExportZkns exports addresses from the VT serving graph to a legacy zkns server. // Note these functions only work with a zktopo. func (wr *Wrangler) ExportZkns(ctx context.Context, cell string) error { - zkTopo, ok := wr.ts.(*zktopo.Server) + zkTopo, ok := wr.ts.Impl.(*zktopo.Server) if !ok { return fmt.Errorf("ExportZkns only works with zktopo") } @@ -55,7 +55,7 @@ func (wr *Wrangler) ExportZkns(ctx context.Context, cell string) error { // ExportZknsForKeyspace exports addresses from the VT serving graph to a legacy zkns server. func (wr *Wrangler) ExportZknsForKeyspace(ctx context.Context, keyspace string) error { - zkTopo, ok := wr.ts.(*zktopo.Server) + zkTopo, ok := wr.ts.Impl.(*zktopo.Server) if !ok { return fmt.Errorf("ExportZknsForKeyspace only works with zktopo") } diff --git a/go/vt/zktopo/server.go b/go/vt/zktopo/server.go index bd18770224..bf45cdd1a2 100644 --- a/go/vt/zktopo/server.go +++ b/go/vt/zktopo/server.go @@ -33,14 +33,14 @@ func (zkts *Server) GetZConn() zk.Conn { // NewServer can be used to create a custom Server // (for tests for instance) but it cannot change the globally // registered one. -func NewServer(zconn zk.Conn) *Server { - return &Server{zconn: zconn} +func NewServer(zconn zk.Conn) topo.Server { + return topo.Server{Impl: &Server{zconn: zconn}} } func init() { zconn := zk.NewMetaConn() stats.PublishJSONFunc("ZkMetaConn", zconn.String) - topo.RegisterServer("zookeeper", NewServer(zconn)) + topo.RegisterServer("zookeeper", &Server{zconn: zconn}) } // diff --git a/go/vt/zktopo/testserver.go b/go/vt/zktopo/testserver.go index 9e837aab44..7dff1b835d 100644 --- a/go/vt/zktopo/testserver.go +++ b/go/vt/zktopo/testserver.go @@ -14,14 +14,14 @@ import ( // TestServer is a proxy for a real implementation of topo.Server that // provides hooks for testing. type TestServer struct { - topo.Server + topo.Impl localCells []string HookLockSrvShardForAction func() } // NewTestServer returns a new TestServer (with the required paths created) -func NewTestServer(t *testing.T, cells []string) *TestServer { +func NewTestServer(t *testing.T, cells []string) topo.Server { zconn := fakezk.NewConn() // create the toplevel zk paths @@ -33,7 +33,7 @@ func NewTestServer(t *testing.T, cells []string) *TestServer { t.Fatalf("cannot init ZooKeeper: %v", err) } } - return &TestServer{Server: NewServer(zconn), localCells: cells} + return topo.Server{Impl: &TestServer{Impl: &Server{zconn}, localCells: cells}} } // GetKnownCells is part of topo.Server interface @@ -47,19 +47,5 @@ func (s *TestServer) LockSrvShardForAction(ctx context.Context, cell, keyspace, if s.HookLockSrvShardForAction != nil { s.HookLockSrvShardForAction() } - return s.Server.LockSrvShardForAction(ctx, cell, keyspace, shard, contents) -} - -// SaveVSchema has to be redefined here. -// Otherwise the test type assertion fails. -// TODO(sougou): Remove these two functions after they're -// migrated into topo.Server. -func (s *TestServer) SaveVSchema(ctx context.Context, vschema string) error { - return s.Server.(topo.Schemafier).SaveVSchema(ctx, vschema) -} - -// GetVSchema has to be redefined here. -// Otherwise the test type assertion fails. -func (s *TestServer) GetVSchema(ctx context.Context) (string, error) { - return s.Server.(topo.Schemafier).GetVSchema(ctx) + return s.Impl.LockSrvShardForAction(ctx, cell, keyspace, shard, contents) } diff --git a/go/vt/zktopo/testserver_test.go b/go/vt/zktopo/testserver_test.go index 01eacc7bc1..cc9faf8531 100644 --- a/go/vt/zktopo/testserver_test.go +++ b/go/vt/zktopo/testserver_test.go @@ -20,7 +20,7 @@ func TestHookLockSrvShardForAction(t *testing.T) { ts := NewTestServer(t, cells) triggered := false - ts.HookLockSrvShardForAction = func() { + ts.Impl.(*TestServer).HookLockSrvShardForAction = func() { triggered = true } diff --git a/go/vt/zktopo/zktopo_test.go b/go/vt/zktopo/zktopo_test.go index 4f21da4191..731c80b3c2 100644 --- a/go/vt/zktopo/zktopo_test.go +++ b/go/vt/zktopo/zktopo_test.go @@ -102,7 +102,7 @@ func TestPurgeActions(t *testing.T) { } actionPath := path.Join(globalKeyspacesPath, "test_keyspace", "action") - zkts := ts.Server.(*Server) + zkts := ts.Impl.(*TestServer).Impl.(*Server) if _, err := zk.CreateRecursive(zkts.zconn, actionPath+"/topurge", "purgeme", 0, zookeeper.WorldACL(zookeeper.PERM_ALL)); err != nil { t.Fatalf("CreateRecursive(topurge): %v", err) @@ -134,7 +134,7 @@ func TestPruneActionLogs(t *testing.T) { } actionLogPath := path.Join(globalKeyspacesPath, "test_keyspace", "actionlog") - zkts := ts.Server.(*Server) + zkts := ts.Impl.(*TestServer).Impl.(*Server) if _, err := zk.CreateRecursive(zkts.zconn, actionLogPath+"/0", "first", 0, zookeeper.WorldACL(zookeeper.PERM_ALL)); err != nil { t.Fatalf("CreateRecursive(stale): %v", err) From 2b965857584c77a4372e9e7f5606528265f12b82 Mon Sep 17 00:00:00 2001 From: Ammar Aijazi Date: Wed, 19 Aug 2015 16:00:06 -0700 Subject: [PATCH 60/80] Suppress or demote some Execute errors in logs --- go/mysql/mysql.go | 3 +++ go/vt/tabletserver/sqlquery.go | 23 ++++++++++++++++++----- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/go/mysql/mysql.go b/go/mysql/mysql.go index 76b44ace1f..ccd7bccbce 100644 --- a/go/mysql/mysql.go +++ b/go/mysql/mysql.go @@ -50,6 +50,9 @@ const ( // ErrOptionPreventsStatement is C.ER_OPTION_PREVENTS_STATEMENT ErrOptionPreventsStatement = C.ER_OPTION_PREVENTS_STATEMENT + // ErrDataTooLong is C.ER_DATA_TOO_LONG + ErrDataTooLong = C.ER_DATA_TOO_LONG + // ErrServerLost is C.CR_SERVER_LOST. // It's hard-coded for now because it causes problems on import. ErrServerLost = 2013 diff --git a/go/vt/tabletserver/sqlquery.go b/go/vt/tabletserver/sqlquery.go index fa0b0c8b21..e52b296f54 100644 --- a/go/vt/tabletserver/sqlquery.go +++ b/go/vt/tabletserver/sqlquery.go @@ -370,14 +370,27 @@ func (sq *SqlQuery) handleExecErrorNoPanic(query *proto.Query, err interface{}, myError = terr } terr.RecordStats(sq.qe.queryServiceStats) - // suppress these errors in logs - if terr.ErrorType == ErrRetry || terr.ErrorType == ErrTxPoolFull || terr.SqlError == mysql.ErrDupEntry { + + logMethod := log.Warningf + // Suppress or demote some errors in logs + switch terr.ErrorType { + case ErrRetry, ErrTxPoolFull: return myError + case ErrFatal: + logMethod = log.Errorf } - if terr.ErrorType == ErrFatal { - log.Errorf("%v: %v", terr, query) + // We want to suppress/demote some MySQL error codes (regardless of the ErrorType) + switch terr.SqlError { + case mysql.ErrDupEntry: + return myError + case mysql.ErrLockWaitTimeout, mysql.ErrLockDeadlock, mysql.ErrDataTooLong: + logMethod = log.Infof + case 0: + if strings.Contains(terr.Error(), "Row count exceeded") { + logMethod = log.Infof + } } - log.Warningf("%v: %v", terr, query) + logMethod("%v: %v", terr, query) return myError } From 4c41ba402e48185e22a1b25112e0a831ed578f0d Mon Sep 17 00:00:00 2001 From: Alain Jobart Date: Wed, 19 Aug 2015 17:59:50 -0700 Subject: [PATCH 61/80] Now passing topo.Impl to unit tests. (still needs to create a topo.Server in a couple places, but that will go away soon). --- go/vt/etcdtopo/explorer_test.go | 8 ++++---- go/vt/etcdtopo/server_test.go | 6 ++---- go/vt/topo/helpers/tee_topo_test.go | 4 ++-- go/vt/topo/test/keyspace.go | 6 +++--- go/vt/topo/test/lock.go | 26 +++++++++++++------------- go/vt/topo/test/replication.go | 4 ++-- go/vt/topo/test/serving.go | 18 +++++++++--------- go/vt/topo/test/shard.go | 20 +++++++++++--------- go/vt/topo/test/tablet.go | 8 ++++---- go/vt/topo/test/testing.go | 6 +++--- go/vt/topo/test/vschema.go | 2 +- go/vt/zktopo/zktopo_test.go | 28 ++++++++++++++-------------- 12 files changed, 68 insertions(+), 68 deletions(-) diff --git a/go/vt/etcdtopo/explorer_test.go b/go/vt/etcdtopo/explorer_test.go index 0202a567a3..71ba35becf 100644 --- a/go/vt/etcdtopo/explorer_test.go +++ b/go/vt/etcdtopo/explorer_test.go @@ -73,7 +73,7 @@ func TestHandlePathRoot(t *testing.T) { want := []string{"global", "cell1", "cell2", "cell3"} ts := newTestServer(t, cells) - ex := NewExplorer(ts.Impl.(*Server)) + ex := NewExplorer(ts.(*Server)) result := ex.HandlePath(nil, input, nil) exResult := result.(*explorerResult) if got := exResult.Children; !reflect.DeepEqual(got, want) { @@ -101,7 +101,7 @@ func TestHandlePathKeyspace(t *testing.T) { } m := &mockActionRepo{} - ex := NewExplorer(ts.Impl.(*Server)) + ex := NewExplorer(ts.(*Server)) result := ex.HandlePath(m, input, nil) exResult := result.(*explorerResult) if got := exResult.Data; got != want { @@ -135,7 +135,7 @@ func TestHandlePathShard(t *testing.T) { } m := &mockActionRepo{} - ex := NewExplorer(ts.Impl.(*Server)) + ex := NewExplorer(ts.(*Server)) result := ex.HandlePath(m, input, nil) exResult := result.(*explorerResult) if got := exResult.Data; got != want { @@ -169,7 +169,7 @@ func TestHandlePathTablet(t *testing.T) { } m := &mockActionRepo{} - ex := NewExplorer(ts.Impl.(*Server)) + ex := NewExplorer(ts.(*Server)) result := ex.HandlePath(m, input, nil) exResult := result.(*explorerResult) if got := exResult.Data; got != want { diff --git a/go/vt/etcdtopo/server_test.go b/go/vt/etcdtopo/server_test.go index ad39f5d240..b80e42333b 100644 --- a/go/vt/etcdtopo/server_test.go +++ b/go/vt/etcdtopo/server_test.go @@ -13,7 +13,7 @@ import ( "golang.org/x/net/context" ) -func newTestServer(t *testing.T, cells []string) topo.Server { +func newTestServer(t *testing.T, cells []string) topo.Impl { s := &Server{ _cells: make(map[string]*cellClient), newClient: newTestClient, @@ -28,9 +28,7 @@ func newTestServer(t *testing.T, cells []string) topo.Server { c.Set("/vt/cells/"+cell, cell, 0) } - return topo.Server{ - Impl: s, - } + return s } func TestKeyspace(t *testing.T) { diff --git a/go/vt/topo/helpers/tee_topo_test.go b/go/vt/topo/helpers/tee_topo_test.go index 57573fefff..4fcacc69e1 100644 --- a/go/vt/topo/helpers/tee_topo_test.go +++ b/go/vt/topo/helpers/tee_topo_test.go @@ -28,7 +28,7 @@ func (s fakeServer) GetKnownCells(ctx context.Context) ([]string, error) { return s.localCells, nil } -func newFakeTeeServer(t *testing.T) topo.Server { +func newFakeTeeServer(t *testing.T) topo.Impl { cells := []string{"test", "global"} // global has to be last zconn1 := fakezk.NewConn() @@ -45,7 +45,7 @@ func newFakeTeeServer(t *testing.T) topo.Server { s1 := fakeServer{Server: zktopo.NewServer(zconn1), localCells: cells[:len(cells)-1]} s2 := fakeServer{Server: zktopo.NewServer(zconn2), localCells: cells[:len(cells)-1]} - return topo.Server{Impl: NewTee(s1, s2, false)} + return NewTee(s1, s2, false) } func TestKeyspace(t *testing.T) { diff --git a/go/vt/topo/test/keyspace.go b/go/vt/topo/test/keyspace.go index 6c490dd120..3522969d58 100644 --- a/go/vt/topo/test/keyspace.go +++ b/go/vt/topo/test/keyspace.go @@ -1,4 +1,4 @@ -// Package test contains utilities to test topo.Server +// Package test contains utilities to test topo.Impl // implementations. If you are testing your implementation, you will // want to call all the check methods in your test methods. For an // example, look at the tests in @@ -16,7 +16,7 @@ import ( ) // CheckKeyspace tests the keyspace part of the API -func CheckKeyspace(ctx context.Context, t *testing.T, ts topo.Server) { +func CheckKeyspace(ctx context.Context, t *testing.T, ts topo.Impl) { keyspaces, err := ts.GetKeyspaces(ctx) if err != nil { t.Errorf("GetKeyspaces(empty): %v", err) @@ -103,7 +103,7 @@ func CheckKeyspace(ctx context.Context, t *testing.T, ts topo.Server) { newServedFroms = append(newServedFroms, ksf) } ki.ServedFroms = newServedFroms - err = topo.UpdateKeyspace(ctx, ts, ki) + err = topo.UpdateKeyspace(ctx, topo.Server{Impl: ts}, ki) if err != nil { t.Fatalf("UpdateKeyspace: %v", err) } diff --git a/go/vt/topo/test/lock.go b/go/vt/topo/test/lock.go index 3dc8712e36..0be6b2ee32 100644 --- a/go/vt/topo/test/lock.go +++ b/go/vt/topo/test/lock.go @@ -1,4 +1,4 @@ -// Package test contains utilities to test topo.Server +// Package test contains utilities to test topo.Impl // implementations. If you are testing your implementation, you will // want to call CheckAll in your test method. For an example, look at // the tests in github.com/youtube/vitess/go/vt/zktopo. @@ -20,7 +20,7 @@ import ( var timeUntilLockIsTaken = 10 * time.Millisecond // CheckKeyspaceLock checks we can take a keyspace lock as expected. -func CheckKeyspaceLock(ctx context.Context, t *testing.T, ts topo.Server) { +func CheckKeyspaceLock(ctx context.Context, t *testing.T, ts topo.Impl) { if err := ts.CreateKeyspace(ctx, "test_keyspace", &pb.Keyspace{}); err != nil { t.Fatalf("CreateKeyspace: %v", err) } @@ -30,7 +30,7 @@ func CheckKeyspaceLock(ctx context.Context, t *testing.T, ts topo.Server) { checkKeyspaceLockUnblocks(ctx, t, ts) } -func checkKeyspaceLockTimeout(ctx context.Context, t *testing.T, ts topo.Server) { +func checkKeyspaceLockTimeout(ctx context.Context, t *testing.T, ts topo.Impl) { lockPath, err := ts.LockKeyspaceForAction(ctx, "test_keyspace", "fake-content") if err != nil { t.Fatalf("LockKeyspaceForAction: %v", err) @@ -64,7 +64,7 @@ func checkKeyspaceLockTimeout(ctx context.Context, t *testing.T, ts topo.Server) } // checkKeyspaceLockMissing makes sure we can't lock a non-existing keyspace -func checkKeyspaceLockMissing(ctx context.Context, t *testing.T, ts topo.Server) { +func checkKeyspaceLockMissing(ctx context.Context, t *testing.T, ts topo.Impl) { if _, err := ts.LockKeyspaceForAction(ctx, "test_keyspace_666", "fake-content"); err == nil { t.Fatalf("LockKeyspaceForAction(test_keyspace_666) worked for non-existing keyspace") } @@ -72,7 +72,7 @@ func checkKeyspaceLockMissing(ctx context.Context, t *testing.T, ts topo.Server) // checkKeyspaceLockUnblocks makes sure that a routine waiting on a lock // is unblocked when another routine frees the lock -func checkKeyspaceLockUnblocks(ctx context.Context, t *testing.T, ts topo.Server) { +func checkKeyspaceLockUnblocks(ctx context.Context, t *testing.T, ts topo.Impl) { unblock := make(chan struct{}) finished := make(chan struct{}) @@ -114,11 +114,11 @@ func checkKeyspaceLockUnblocks(ctx context.Context, t *testing.T, ts topo.Server } // CheckShardLock checks we can take a shard lock -func CheckShardLock(ctx context.Context, t *testing.T, ts topo.Server) { +func CheckShardLock(ctx context.Context, t *testing.T, ts topo.Impl) { if err := ts.CreateKeyspace(ctx, "test_keyspace", &pb.Keyspace{}); err != nil { t.Fatalf("CreateKeyspace: %v", err) } - if err := topo.CreateShard(ctx, ts, "test_keyspace", "10-20"); err != nil { + if err := topo.CreateShard(ctx, topo.Server{Impl: ts}, "test_keyspace", "10-20"); err != nil { t.Fatalf("CreateShard: %v", err) } @@ -127,7 +127,7 @@ func CheckShardLock(ctx context.Context, t *testing.T, ts topo.Server) { checkShardLockUnblocks(ctx, t, ts) } -func checkShardLockTimeout(ctx context.Context, t *testing.T, ts topo.Server) { +func checkShardLockTimeout(ctx context.Context, t *testing.T, ts topo.Impl) { lockPath, err := ts.LockShardForAction(ctx, "test_keyspace", "10-20", "fake-content") if err != nil { t.Fatalf("LockShardForAction: %v", err) @@ -160,7 +160,7 @@ func checkShardLockTimeout(ctx context.Context, t *testing.T, ts topo.Server) { } } -func checkShardLockMissing(ctx context.Context, t *testing.T, ts topo.Server) { +func checkShardLockMissing(ctx context.Context, t *testing.T, ts topo.Impl) { // test we can't lock a non-existing shard if _, err := ts.LockShardForAction(ctx, "test_keyspace", "20-30", "fake-content"); err == nil { t.Fatalf("LockShardForAction(test_keyspace/20-30) worked for non-existing shard") @@ -169,7 +169,7 @@ func checkShardLockMissing(ctx context.Context, t *testing.T, ts topo.Server) { // checkShardLockUnblocks makes sure that a routine waiting on a lock // is unblocked when another routine frees the lock -func checkShardLockUnblocks(ctx context.Context, t *testing.T, ts topo.Server) { +func checkShardLockUnblocks(ctx context.Context, t *testing.T, ts topo.Impl) { unblock := make(chan struct{}) finished := make(chan struct{}) @@ -211,12 +211,12 @@ func checkShardLockUnblocks(ctx context.Context, t *testing.T, ts topo.Server) { } // CheckSrvShardLock tests we can take a SrvShard lock -func CheckSrvShardLock(ctx context.Context, t *testing.T, ts topo.Server) { +func CheckSrvShardLock(ctx context.Context, t *testing.T, ts topo.Impl) { checkSrvShardLockGeneral(ctx, t, ts) checkSrvShardLockUnblocks(ctx, t, ts) } -func checkSrvShardLockGeneral(ctx context.Context, t *testing.T, ts topo.Server) { +func checkSrvShardLockGeneral(ctx context.Context, t *testing.T, ts topo.Impl) { cell := getLocalCell(ctx, t, ts) // make sure we can create the lock even if no directory exists @@ -266,7 +266,7 @@ func checkSrvShardLockGeneral(ctx context.Context, t *testing.T, ts topo.Server) // checkSrvShardLockUnblocks makes sure that a routine waiting on a lock // is unblocked when another routine frees the lock -func checkSrvShardLockUnblocks(ctx context.Context, t *testing.T, ts topo.Server) { +func checkSrvShardLockUnblocks(ctx context.Context, t *testing.T, ts topo.Impl) { cell := getLocalCell(ctx, t, ts) unblock := make(chan struct{}) finished := make(chan struct{}) diff --git a/go/vt/topo/test/replication.go b/go/vt/topo/test/replication.go index 2d14d9ac6a..9870fe6406 100644 --- a/go/vt/topo/test/replication.go +++ b/go/vt/topo/test/replication.go @@ -1,4 +1,4 @@ -// Package test contains utilities to test topo.Server +// Package test contains utilities to test topo.Impl // implementations. If you are testing your implementation, you will // want to call CheckAll in your test method. For an example, look at // the tests in github.com/youtube/vitess/go/vt/zktopo. @@ -14,7 +14,7 @@ import ( ) // CheckShardReplication tests ShardReplication objects -func CheckShardReplication(ctx context.Context, t *testing.T, ts topo.Server) { +func CheckShardReplication(ctx context.Context, t *testing.T, ts topo.Impl) { cell := getLocalCell(ctx, t, ts) if _, err := ts.GetShardReplication(ctx, cell, "test_keyspace", "-10"); err != topo.ErrNoNode { t.Errorf("GetShardReplication(not there): %v", err) diff --git a/go/vt/topo/test/serving.go b/go/vt/topo/test/serving.go index 523bf1bd95..848349b79f 100644 --- a/go/vt/topo/test/serving.go +++ b/go/vt/topo/test/serving.go @@ -1,4 +1,4 @@ -// Package test contains utilities to test topo.Server +// Package test contains utilities to test topo.Impl // implementations. If you are testing your implementation, you will // want to call CheckAll in your test method. For an example, look at // the tests in github.com/youtube/vitess/go/vt/zktopo. @@ -16,7 +16,7 @@ import ( ) // CheckServingGraph makes sure the serving graph functions work properly. -func CheckServingGraph(ctx context.Context, t *testing.T, ts topo.Server) { +func CheckServingGraph(ctx context.Context, t *testing.T, ts topo.Impl) { cell := getLocalCell(ctx, t, ts) // test individual cell/keyspace/shard/type entries @@ -57,7 +57,7 @@ func CheckServingGraph(ctx context.Context, t *testing.T, ts topo.Server) { // Make a change. tmp := endPoints.Entries[0].Uid endPoints.Entries[0].Uid = tmp + 1 - if err := topo.UpdateEndPoints(ctx, ts, cell, "test_keyspace", "-10", pb.TabletType_MASTER, endPoints, -1); err != nil { + if err := ts.UpdateEndPoints(ctx, cell, "test_keyspace", "-10", pb.TabletType_MASTER, endPoints, -1); err != nil { t.Fatalf("UpdateEndPoints(master): %v", err) } endPoints.Entries[0].Uid = tmp @@ -74,7 +74,7 @@ func CheckServingGraph(ctx context.Context, t *testing.T, ts topo.Server) { t.Fatalf("DeleteEndPoints: %v", err) } // Recreate it with an unconditional update. - if err := topo.UpdateEndPoints(ctx, ts, cell, "test_keyspace", "-10", pb.TabletType_MASTER, endPoints, -1); err != nil { + if err := ts.UpdateEndPoints(ctx, cell, "test_keyspace", "-10", pb.TabletType_MASTER, endPoints, -1); err != nil { t.Fatalf("UpdateEndPoints(master): %v", err) } @@ -96,7 +96,7 @@ func CheckServingGraph(ctx context.Context, t *testing.T, ts topo.Server) { } // Re-add endpoints. - if err := topo.UpdateEndPoints(ctx, ts, cell, "test_keyspace", "-10", pb.TabletType_MASTER, endPoints, -1); err != nil { + if err := ts.UpdateEndPoints(ctx, cell, "test_keyspace", "-10", pb.TabletType_MASTER, endPoints, -1); err != nil { t.Fatalf("UpdateEndPoints(master): %v", err) } @@ -112,15 +112,15 @@ func CheckServingGraph(ctx context.Context, t *testing.T, ts topo.Server) { } // Update with the wrong version. - if err := topo.UpdateEndPoints(ctx, ts, cell, "test_keyspace", "-10", pb.TabletType_MASTER, endPoints, version+1); err != topo.ErrBadVersion { + if err := ts.UpdateEndPoints(ctx, cell, "test_keyspace", "-10", pb.TabletType_MASTER, endPoints, version+1); err != topo.ErrBadVersion { t.Fatalf("UpdateEndPoints(master): err = %v, want topo.ErrBadVersion", err) } // Update with the right version. - if err := topo.UpdateEndPoints(ctx, ts, cell, "test_keyspace", "-10", pb.TabletType_MASTER, endPoints, version); err != nil { + if err := ts.UpdateEndPoints(ctx, cell, "test_keyspace", "-10", pb.TabletType_MASTER, endPoints, version); err != nil { t.Fatalf("UpdateEndPoints(master): %v", err) } // Update existing EndPoints unconditionally. - if err := topo.UpdateEndPoints(ctx, ts, cell, "test_keyspace", "-10", pb.TabletType_MASTER, endPoints, -1); err != nil { + if err := ts.UpdateEndPoints(ctx, cell, "test_keyspace", "-10", pb.TabletType_MASTER, endPoints, -1); err != nil { t.Fatalf("UpdateEndPoints(master): %v", err) } @@ -213,7 +213,7 @@ func CheckServingGraph(ctx context.Context, t *testing.T, ts topo.Server) { } // CheckWatchSrvKeyspace makes sure WatchSrvKeyspace works as expected -func CheckWatchSrvKeyspace(ctx context.Context, t *testing.T, ts topo.Server) { +func CheckWatchSrvKeyspace(ctx context.Context, t *testing.T, ts topo.Impl) { cell := getLocalCell(ctx, t, ts) keyspace := "test_keyspace" diff --git a/go/vt/topo/test/shard.go b/go/vt/topo/test/shard.go index e6331d9fec..53ed96fdc5 100644 --- a/go/vt/topo/test/shard.go +++ b/go/vt/topo/test/shard.go @@ -1,4 +1,4 @@ -// Package test contains utilities to test topo.Server +// Package test contains utilities to test topo.Impl // implementations. If you are testing your implementation, you will // want to call CheckAll in your test method. For an example, look at // the tests in github.com/youtube/vitess/go/vt/zktopo. @@ -29,15 +29,17 @@ func shardEqual(left, right *pb.Shard) (bool, error) { } // CheckShard verifies the Shard operations work correctly -func CheckShard(ctx context.Context, t *testing.T, ts topo.Server) { +func CheckShard(ctx context.Context, t *testing.T, ts topo.Impl) { + tts := topo.Server{Impl: ts} + if err := ts.CreateKeyspace(ctx, "test_keyspace", &pb.Keyspace{}); err != nil { t.Fatalf("CreateKeyspace: %v", err) } - if err := topo.CreateShard(ctx, ts, "test_keyspace", "b0-c0"); err != nil { + if err := topo.CreateShard(ctx, tts, "test_keyspace", "b0-c0"); err != nil { t.Fatalf("CreateShard: %v", err) } - if err := topo.CreateShard(ctx, ts, "test_keyspace", "b0-c0"); err != topo.ErrNodeExists { + if err := topo.CreateShard(ctx, tts, "test_keyspace", "b0-c0"); err != topo.ErrNodeExists { t.Errorf("CreateShard called second time, got: %v", err) } @@ -45,7 +47,7 @@ func CheckShard(ctx context.Context, t *testing.T, ts topo.Server) { if err := ts.DeleteShard(ctx, "test_keyspace", "b0-c0"); err != nil { t.Fatalf("DeleteShard: %v", err) } - if err := topo.CreateShard(ctx, ts, "test_keyspace", "b0-c0"); err != nil { + if err := topo.CreateShard(ctx, tts, "test_keyspace", "b0-c0"); err != nil { t.Fatalf("CreateShard: %v", err) } @@ -53,7 +55,7 @@ func CheckShard(ctx context.Context, t *testing.T, ts topo.Server) { if err := ts.DeleteKeyspaceShards(ctx, "test_keyspace"); err != nil { t.Fatalf("DeleteKeyspaceShards: %v", err) } - if err := topo.CreateShard(ctx, ts, "test_keyspace", "b0-c0"); err != nil { + if err := topo.CreateShard(ctx, tts, "test_keyspace", "b0-c0"); err != nil { t.Fatalf("CreateShard: %v", err) } @@ -103,12 +105,12 @@ func CheckShard(ctx context.Context, t *testing.T, ts topo.Server) { DisableQueryService: true, }, } - if err := topo.UpdateShard(ctx, ts, shardInfo); err != nil { + if err := topo.UpdateShard(ctx, tts, shardInfo); err != nil { t.Errorf("UpdateShard: %v", err) } other := &pb.TabletAlias{Cell: "ny", Uid: 82873} - _, err = topo.UpdateShardFields(ctx, ts, "test_keyspace", "b0-c0", func(shard *pb.Shard) error { + _, err = topo.UpdateShardFields(ctx, tts, "test_keyspace", "b0-c0", func(shard *pb.Shard) error { shard.MasterAlias = other return nil }) @@ -122,7 +124,7 @@ func CheckShard(ctx context.Context, t *testing.T, ts topo.Server) { if *si.MasterAlias != *other { t.Fatalf("shard.MasterAlias = %v, want %v", si.MasterAlias, other) } - _, err = topo.UpdateShardFields(ctx, ts, "test_keyspace", "b0-c0", func(shard *pb.Shard) error { + _, err = topo.UpdateShardFields(ctx, tts, "test_keyspace", "b0-c0", func(shard *pb.Shard) error { shard.MasterAlias = master return nil }) diff --git a/go/vt/topo/test/tablet.go b/go/vt/topo/test/tablet.go index f6113b5c2b..a36a072d5b 100644 --- a/go/vt/topo/test/tablet.go +++ b/go/vt/topo/test/tablet.go @@ -1,4 +1,4 @@ -// Package test contains utilities to test topo.Server +// Package test contains utilities to test topo.Impl // implementations. If you are testing your implementation, you will // want to call CheckAll in your test method. For an example, look at // the tests in github.com/youtube/vitess/go/vt/zktopo. @@ -28,7 +28,7 @@ func tabletEqual(left, right *pb.Tablet) (bool, error) { } // CheckTablet verifies the topo server API is correct for managing tablets. -func CheckTablet(ctx context.Context, t *testing.T, ts topo.Server) { +func CheckTablet(ctx context.Context, t *testing.T, ts topo.Impl) { cell := getLocalCell(ctx, t, ts) tablet := &pb.Tablet{ Alias: &pb.TabletAlias{Cell: cell, Uid: 1}, @@ -78,7 +78,7 @@ func CheckTablet(ctx context.Context, t *testing.T, ts topo.Server) { } ti.Hostname = "remotehost" - if err := topo.UpdateTablet(ctx, ts, ti); err != nil { + if err := topo.UpdateTablet(ctx, topo.Server{Impl: ts}, ti); err != nil { t.Errorf("UpdateTablet: %v", err) } @@ -90,7 +90,7 @@ func CheckTablet(ctx context.Context, t *testing.T, ts topo.Server) { t.Errorf("ti.Hostname: want %v, got %v", want, ti.Hostname) } - if err := topo.UpdateTabletFields(ctx, ts, tablet.Alias, func(t *pb.Tablet) error { + if err := topo.UpdateTabletFields(ctx, topo.Server{Impl: ts}, tablet.Alias, func(t *pb.Tablet) error { t.Hostname = "anotherhost" return nil }); err != nil { diff --git a/go/vt/topo/test/testing.go b/go/vt/topo/test/testing.go index dc0d54881b..def30f719a 100644 --- a/go/vt/topo/test/testing.go +++ b/go/vt/topo/test/testing.go @@ -1,4 +1,4 @@ -// Package test contains utilities to test topo.Server +// Package test contains utilities to test topo.Impl // implementations. If you are testing your implementation, you will // want to call CheckAll in your test method. For an example, look at // the tests in github.com/youtube/vitess/go/vt/zktopo. @@ -30,13 +30,13 @@ func newKeyRange3(value string) *pb.KeyRange { return result } -func getLocalCell(ctx context.Context, t *testing.T, ts topo.Server) string { +func getLocalCell(ctx context.Context, t *testing.T, ts topo.Impl) string { cells, err := ts.GetKnownCells(ctx) if err != nil { t.Fatalf("GetKnownCells: %v", err) } if len(cells) < 1 { - t.Fatalf("provided topo.Server doesn't have enough cells (need at least 1): %v", cells) + t.Fatalf("provided topo.Impl doesn't have enough cells (need at least 1): %v", cells) } return cells[0] } diff --git a/go/vt/topo/test/vschema.go b/go/vt/topo/test/vschema.go index 3405bc376d..db48d30f3e 100644 --- a/go/vt/topo/test/vschema.go +++ b/go/vt/topo/test/vschema.go @@ -13,7 +13,7 @@ import ( ) // CheckVSchema runs the tests on the VSchema part of the API -func CheckVSchema(ctx context.Context, t *testing.T, ts topo.Server) { +func CheckVSchema(ctx context.Context, t *testing.T, ts topo.Impl) { got, err := ts.GetVSchema(ctx) if err != nil { t.Error(err) diff --git a/go/vt/zktopo/zktopo_test.go b/go/vt/zktopo/zktopo_test.go index 731c80b3c2..d581445c87 100644 --- a/go/vt/zktopo/zktopo_test.go +++ b/go/vt/zktopo/zktopo_test.go @@ -16,48 +16,48 @@ import ( func TestKeyspace(t *testing.T) { ctx := context.Background() - ts := NewTestServer(t, []string{"test"}) + ts := NewTestServer(t, []string{"test"}).Impl defer ts.Close() test.CheckKeyspace(ctx, t, ts) } func TestShard(t *testing.T) { ctx := context.Background() - ts := NewTestServer(t, []string{"test"}) + ts := NewTestServer(t, []string{"test"}).Impl defer ts.Close() test.CheckShard(ctx, t, ts) } func TestTablet(t *testing.T) { ctx := context.Background() - ts := NewTestServer(t, []string{"test"}) + ts := NewTestServer(t, []string{"test"}).Impl defer ts.Close() test.CheckTablet(ctx, t, ts) } func TestShardReplication(t *testing.T) { ctx := context.Background() - ts := NewTestServer(t, []string{"test"}) + ts := NewTestServer(t, []string{"test"}).Impl defer ts.Close() test.CheckShardReplication(ctx, t, ts) } func TestServingGraph(t *testing.T) { - ts := NewTestServer(t, []string{"test"}) + ts := NewTestServer(t, []string{"test"}).Impl defer ts.Close() test.CheckServingGraph(context.Background(), t, ts) } func TestWatchSrvKeyspace(t *testing.T) { WatchSleepDuration = 2 * time.Millisecond - ts := NewTestServer(t, []string{"test"}) + ts := NewTestServer(t, []string{"test"}).Impl defer ts.Close() test.CheckWatchSrvKeyspace(context.Background(), t, ts) } func TestKeyspaceLock(t *testing.T) { ctx := context.Background() - ts := NewTestServer(t, []string{"test"}) + ts := NewTestServer(t, []string{"test"}).Impl defer ts.Close() test.CheckKeyspaceLock(ctx, t, ts) } @@ -68,7 +68,7 @@ func TestShardLock(t *testing.T) { t.Skip("skipping wait-based test in short mode.") } - ts := NewTestServer(t, []string{"test"}) + ts := NewTestServer(t, []string{"test"}).Impl defer ts.Close() test.CheckShardLock(ctx, t, ts) } @@ -79,14 +79,14 @@ func TestSrvShardLock(t *testing.T) { t.Skip("skipping wait-based test in short mode.") } - ts := NewTestServer(t, []string{"test"}) + ts := NewTestServer(t, []string{"test"}).Impl defer ts.Close() test.CheckSrvShardLock(ctx, t, ts) } func TestVSchema(t *testing.T) { ctx := context.Background() - ts := NewTestServer(t, []string{"test"}) + ts := NewTestServer(t, []string{"test"}).Impl defer ts.Close() test.CheckVSchema(ctx, t, ts) } @@ -94,7 +94,7 @@ func TestVSchema(t *testing.T) { // TestPurgeActions is a ZK specific unit test func TestPurgeActions(t *testing.T) { ctx := context.Background() - ts := NewTestServer(t, []string{"test"}) + ts := NewTestServer(t, []string{"test"}).Impl defer ts.Close() if err := ts.CreateKeyspace(ctx, "test_keyspace", &pb.Keyspace{}); err != nil { @@ -102,7 +102,7 @@ func TestPurgeActions(t *testing.T) { } actionPath := path.Join(globalKeyspacesPath, "test_keyspace", "action") - zkts := ts.Impl.(*TestServer).Impl.(*Server) + zkts := ts.(*TestServer).Impl.(*Server) if _, err := zk.CreateRecursive(zkts.zconn, actionPath+"/topurge", "purgeme", 0, zookeeper.WorldACL(zookeeper.PERM_ALL)); err != nil { t.Fatalf("CreateRecursive(topurge): %v", err) @@ -126,7 +126,7 @@ func TestPurgeActions(t *testing.T) { // TestPruneActionLogs is a ZK specific unit test func TestPruneActionLogs(t *testing.T) { ctx := context.Background() - ts := NewTestServer(t, []string{"test"}) + ts := NewTestServer(t, []string{"test"}).Impl defer ts.Close() if err := ts.CreateKeyspace(ctx, "test_keyspace", &pb.Keyspace{}); err != nil { @@ -134,7 +134,7 @@ func TestPruneActionLogs(t *testing.T) { } actionLogPath := path.Join(globalKeyspacesPath, "test_keyspace", "actionlog") - zkts := ts.Impl.(*TestServer).Impl.(*Server) + zkts := ts.(*TestServer).Impl.(*Server) if _, err := zk.CreateRecursive(zkts.zconn, actionLogPath+"/0", "first", 0, zookeeper.WorldACL(zookeeper.PERM_ALL)); err != nil { t.Fatalf("CreateRecursive(stale): %v", err) From 7c7ca54110f1a74c5e935780b25042f2c5574f87 Mon Sep 17 00:00:00 2001 From: Alain Jobart Date: Wed, 19 Aug 2015 18:42:00 -0700 Subject: [PATCH 62/80] Updating a couple keyspace methods. More to come. --- go/cmd/topo2topo/topo2topo.go | 8 +++---- go/cmd/vtgate/plugin_fakezk.go | 2 +- go/vt/etcdtopo/keyspace.go | 18 +++----------- go/vt/topo/helpers/copy.go | 10 ++++---- go/vt/topo/helpers/copy_test.go | 10 ++++---- go/vt/topo/helpers/tee.go | 26 ++++++++++---------- go/vt/topo/helpers/tee_topo_test.go | 6 ++--- go/vt/topo/keyspace.go | 36 +++++++++++++++++++++++----- go/vt/topo/server.go | 2 +- go/vt/topo/shard.go | 2 +- go/vt/topo/test/faketopo/faketopo.go | 2 +- go/vt/topo/test/keyspace.go | 2 +- go/vt/topotools/split.go | 2 +- go/vt/vtctl/vtctl.go | 2 +- go/vt/wrangler/keyspace.go | 8 +++---- go/vt/wrangler/rebuild.go | 4 ++-- go/vt/zktopo/keyspace.go | 17 +++---------- 17 files changed, 79 insertions(+), 78 deletions(-) diff --git a/go/cmd/topo2topo/topo2topo.go b/go/cmd/topo2topo/topo2topo.go index c08b45c78f..e2f7591e97 100644 --- a/go/cmd/topo2topo/topo2topo.go +++ b/go/cmd/topo2topo/topo2topo.go @@ -47,15 +47,15 @@ func main() { toTS := topo.GetServerByName(*toTopo) if *doKeyspaces { - helpers.CopyKeyspaces(ctx, fromTS, toTS) + helpers.CopyKeyspaces(ctx, fromTS.Impl, toTS.Impl) } if *doShards { - helpers.CopyShards(ctx, fromTS, toTS, *deleteKeyspaceShards) + helpers.CopyShards(ctx, fromTS.Impl, toTS.Impl, *deleteKeyspaceShards) } if *doShardReplications { - helpers.CopyShardReplications(ctx, fromTS, toTS) + helpers.CopyShardReplications(ctx, fromTS.Impl, toTS.Impl) } if *doTablets { - helpers.CopyTablets(ctx, fromTS, toTS) + helpers.CopyTablets(ctx, fromTS.Impl, toTS.Impl) } } diff --git a/go/cmd/vtgate/plugin_fakezk.go b/go/cmd/vtgate/plugin_fakezk.go index 5be7c7d463..f38325961b 100644 --- a/go/cmd/vtgate/plugin_fakezk.go +++ b/go/cmd/vtgate/plugin_fakezk.go @@ -18,6 +18,6 @@ var fakezkConfig = flag.String("fakezk-config", "", "If set, will read the json func init() { if *fakezkConfig != "" { - topo.RegisterServer("fakezk", zktopo.NewServer(fakezk.NewConnFromFile(*fakezkConfig))) + topo.RegisterServer("fakezk", zktopo.NewServer(fakezk.NewConnFromFile(*fakezkConfig)).Impl) } } diff --git a/go/vt/etcdtopo/keyspace.go b/go/vt/etcdtopo/keyspace.go index db34e9cc81..307f7e374c 100644 --- a/go/vt/etcdtopo/keyspace.go +++ b/go/vt/etcdtopo/keyspace.go @@ -42,13 +42,13 @@ func (s *Server) CreateKeyspace(ctx context.Context, keyspace string, value *pb. } // UpdateKeyspace implements topo.Server. -func (s *Server) UpdateKeyspace(ctx context.Context, ki *topo.KeyspaceInfo, existingVersion int64) (int64, error) { - data, err := json.MarshalIndent(ki.Keyspace, "", " ") +func (s *Server) UpdateKeyspace(ctx context.Context, keyspace string, value *pb.Keyspace, existingVersion int64) (int64, error) { + data, err := json.MarshalIndent(value, "", " ") if err != nil { return -1, err } - resp, err := s.getGlobal().CompareAndSwap(keyspaceFilePath(ki.KeyspaceName()), + resp, err := s.getGlobal().CompareAndSwap(keyspaceFilePath(keyspace), string(data), 0 /* ttl */, "" /* prevValue */, uint64(existingVersion)) if err != nil { return -1, convertError(err) @@ -56,12 +56,6 @@ func (s *Server) UpdateKeyspace(ctx context.Context, ki *topo.KeyspaceInfo, exis if resp.Node == nil { return -1, ErrBadResponse } - - event.Dispatch(&events.KeyspaceChange{ - KeyspaceName: ki.KeyspaceName(), - Keyspace: ki.Keyspace, - Status: "updated", - }) return int64(resp.Node.ModifiedIndex), nil } @@ -119,12 +113,6 @@ func (s *Server) DeleteKeyspaceShards(ctx context.Context, keyspace string) erro if err = rec.Error(); err != nil { return err } - - event.Dispatch(&events.KeyspaceChange{ - KeyspaceName: keyspace, - Keyspace: nil, - Status: "deleted all shards", - }) return nil } diff --git a/go/vt/topo/helpers/copy.go b/go/vt/topo/helpers/copy.go index 2e2703569e..98222fe47a 100644 --- a/go/vt/topo/helpers/copy.go +++ b/go/vt/topo/helpers/copy.go @@ -19,7 +19,7 @@ import ( ) // CopyKeyspaces will create the keyspaces in the destination topo -func CopyKeyspaces(ctx context.Context, fromTS, toTS topo.Server) { +func CopyKeyspaces(ctx context.Context, fromTS, toTS topo.Impl) { keyspaces, err := fromTS.GetKeyspaces(ctx) if err != nil { log.Fatalf("GetKeyspaces: %v", err) @@ -54,7 +54,7 @@ func CopyKeyspaces(ctx context.Context, fromTS, toTS topo.Server) { } // CopyShards will create the shards in the destination topo -func CopyShards(ctx context.Context, fromTS, toTS topo.Server, deleteKeyspaceShards bool) { +func CopyShards(ctx context.Context, fromTS, toTS topo.Impl, deleteKeyspaceShards bool) { keyspaces, err := fromTS.GetKeyspaces(ctx) if err != nil { log.Fatalf("fromTS.GetKeyspaces: %v", err) @@ -83,7 +83,7 @@ func CopyShards(ctx context.Context, fromTS, toTS topo.Server, deleteKeyspaceSha wg.Add(1) go func(keyspace, shard string) { defer wg.Done() - if err := topo.CreateShard(ctx, toTS, keyspace, shard); err != nil { + if err := topo.CreateShard(ctx, topo.Server{Impl: toTS}, keyspace, shard); err != nil { if err == topo.ErrNodeExists { log.Warningf("shard %v/%v already exists", keyspace, shard) } else { @@ -118,7 +118,7 @@ func CopyShards(ctx context.Context, fromTS, toTS topo.Server, deleteKeyspaceSha } // CopyTablets will create the tablets in the destination topo -func CopyTablets(ctx context.Context, fromTS, toTS topo.Server) { +func CopyTablets(ctx context.Context, fromTS, toTS topo.Impl) { cells, err := fromTS.GetKnownCells(ctx) if err != nil { log.Fatalf("fromTS.GetKnownCells: %v", err) @@ -173,7 +173,7 @@ func CopyTablets(ctx context.Context, fromTS, toTS topo.Server) { // CopyShardReplications will create the ShardReplication objects in // the destination topo -func CopyShardReplications(ctx context.Context, fromTS, toTS topo.Server) { +func CopyShardReplications(ctx context.Context, fromTS, toTS topo.Impl) { keyspaces, err := fromTS.GetKeyspaces(ctx) if err != nil { log.Fatalf("fromTS.GetKeyspaces: %v", err) diff --git a/go/vt/topo/helpers/copy_test.go b/go/vt/topo/helpers/copy_test.go index 6735bf33f1..c01e373822 100644 --- a/go/vt/topo/helpers/copy_test.go +++ b/go/vt/topo/helpers/copy_test.go @@ -20,12 +20,12 @@ import ( pb "github.com/youtube/vitess/go/vt/proto/topodata" ) -func createSetup(ctx context.Context, t *testing.T) (topo.Server, topo.Server) { +func createSetup(ctx context.Context, t *testing.T) (topo.Impl, topo.Impl) { fromConn := fakezk.NewConn() - fromTS := zktopo.NewServer(fromConn) + fromTS := zktopo.NewServer(fromConn).Impl toConn := fakezk.NewConn() - toTS := zktopo.NewServer(toConn) + toTS := zktopo.NewServer(toConn).Impl for _, zkPath := range []string{"/zk/test_cell/vt", "/zk/global/vt"} { if _, err := zk.CreateRecursive(fromConn, zkPath, "", 0, zookeeper.WorldACL(zookeeper.PERM_ALL)); err != nil { @@ -40,7 +40,7 @@ func createSetup(ctx context.Context, t *testing.T) (topo.Server, topo.Server) { if err := fromTS.CreateShard(ctx, "test_keyspace", "0", &pb.Shard{Cells: []string{"test_cell"}}); err != nil { t.Fatalf("cannot create shard: %v", err) } - if err := topo.CreateTablet(ctx, fromTS, &pb.Tablet{ + if err := topo.CreateTablet(ctx, topo.Server{Impl: fromTS}, &pb.Tablet{ Alias: &pb.TabletAlias{ Cell: "test_cell", Uid: 123, @@ -60,7 +60,7 @@ func createSetup(ctx context.Context, t *testing.T) (topo.Server, topo.Server) { }); err != nil { t.Fatalf("cannot create master tablet: %v", err) } - if err := topo.CreateTablet(ctx, fromTS, &pb.Tablet{ + if err := topo.CreateTablet(ctx, topo.Server{Impl: fromTS}, &pb.Tablet{ Alias: &pb.TabletAlias{ Cell: "test_cell", Uid: 234, diff --git a/go/vt/topo/helpers/tee.go b/go/vt/topo/helpers/tee.go index b406a4ecb8..1710054c6a 100644 --- a/go/vt/topo/helpers/tee.go +++ b/go/vt/topo/helpers/tee.go @@ -116,8 +116,8 @@ func (tee *Tee) CreateKeyspace(ctx context.Context, keyspace string, value *pb.K } // UpdateKeyspace is part of the topo.Server interface -func (tee *Tee) UpdateKeyspace(ctx context.Context, ki *topo.KeyspaceInfo, existingVersion int64) (newVersion int64, err error) { - if newVersion, err = tee.primary.UpdateKeyspace(ctx, ki, existingVersion); err != nil { +func (tee *Tee) UpdateKeyspace(ctx context.Context, keyspace string, value *pb.Keyspace, existingVersion int64) (newVersion int64, err error) { + if newVersion, err = tee.primary.UpdateKeyspace(ctx, keyspace, value, existingVersion); err != nil { // failed on primary, not updating secondary return } @@ -126,27 +126,27 @@ func (tee *Tee) UpdateKeyspace(ctx context.Context, ki *topo.KeyspaceInfo, exist // and keyspace version in second topo, replace the version number. // if not, this will probably fail and log. tee.mu.Lock() - kvm, ok := tee.keyspaceVersionMapping[ki.KeyspaceName()] + kvm, ok := tee.keyspaceVersionMapping[keyspace] if ok && kvm.readFromVersion == existingVersion { existingVersion = kvm.readFromSecondVersion - delete(tee.keyspaceVersionMapping, ki.KeyspaceName()) + delete(tee.keyspaceVersionMapping, keyspace) } tee.mu.Unlock() - if newVersion2, serr := tee.secondary.UpdateKeyspace(ctx, ki, existingVersion); serr != nil { + if newVersion2, serr := tee.secondary.UpdateKeyspace(ctx, keyspace, value, existingVersion); serr != nil { // not critical enough to fail if serr == topo.ErrNoNode { // the keyspace doesn't exist on the secondary, let's // just create it - if serr = tee.secondary.CreateKeyspace(ctx, ki.KeyspaceName(), ki.Keyspace); serr != nil { - log.Warningf("secondary.CreateKeyspace(%v) failed (after UpdateKeyspace returned ErrNoNode): %v", ki.KeyspaceName(), serr) + if serr = tee.secondary.CreateKeyspace(ctx, keyspace, value); serr != nil { + log.Warningf("secondary.CreateKeyspace(%v) failed (after UpdateKeyspace returned ErrNoNode): %v", keyspace, serr) } else { - log.Infof("secondary.UpdateKeyspace(%v) failed with ErrNoNode, CreateKeyspace then worked.", ki.KeyspaceName()) - ki, gerr := tee.secondary.GetKeyspace(ctx, ki.KeyspaceName()) + log.Infof("secondary.UpdateKeyspace(%v) failed with ErrNoNode, CreateKeyspace then worked.", keyspace) + ki, gerr := tee.secondary.GetKeyspace(ctx, keyspace) if gerr != nil { - log.Warningf("Failed to re-read keyspace(%v) after creating it on secondary: %v", ki.KeyspaceName(), gerr) + log.Warningf("Failed to re-read keyspace(%v) after creating it on secondary: %v", keyspace, gerr) } else { tee.mu.Lock() - tee.keyspaceVersionMapping[ki.KeyspaceName()] = versionMapping{ + tee.keyspaceVersionMapping[keyspace] = versionMapping{ readFromVersion: newVersion, readFromSecondVersion: ki.Version(), } @@ -154,11 +154,11 @@ func (tee *Tee) UpdateKeyspace(ctx context.Context, ki *topo.KeyspaceInfo, exist } } } else { - log.Warningf("secondary.UpdateKeyspace(%v) failed: %v", ki.KeyspaceName(), serr) + log.Warningf("secondary.UpdateKeyspace(%v) failed: %v", keyspace, serr) } } else { tee.mu.Lock() - tee.keyspaceVersionMapping[ki.KeyspaceName()] = versionMapping{ + tee.keyspaceVersionMapping[keyspace] = versionMapping{ readFromVersion: newVersion, readFromSecondVersion: newVersion2, } diff --git a/go/vt/topo/helpers/tee_topo_test.go b/go/vt/topo/helpers/tee_topo_test.go index 4fcacc69e1..ca00966440 100644 --- a/go/vt/topo/helpers/tee_topo_test.go +++ b/go/vt/topo/helpers/tee_topo_test.go @@ -20,7 +20,7 @@ import ( ) type fakeServer struct { - topo.Server + topo.Impl localCells []string } @@ -42,8 +42,8 @@ func newFakeTeeServer(t *testing.T) topo.Impl { t.Fatalf("cannot init ZooKeeper: %v", err) } } - s1 := fakeServer{Server: zktopo.NewServer(zconn1), localCells: cells[:len(cells)-1]} - s2 := fakeServer{Server: zktopo.NewServer(zconn2), localCells: cells[:len(cells)-1]} + s1 := fakeServer{Impl: zktopo.NewServer(zconn1).Impl, localCells: cells[:len(cells)-1]} + s2 := fakeServer{Impl: zktopo.NewServer(zconn2).Impl, localCells: cells[:len(cells)-1]} return NewTee(s1, s2, false) } diff --git a/go/vt/topo/keyspace.go b/go/vt/topo/keyspace.go index d775e222bd..3002301826 100644 --- a/go/vt/topo/keyspace.go +++ b/go/vt/topo/keyspace.go @@ -11,7 +11,9 @@ import ( log "github.com/golang/glog" "golang.org/x/net/context" + "github.com/youtube/vitess/go/event" "github.com/youtube/vitess/go/vt/concurrency" + "github.com/youtube/vitess/go/vt/topo/events" pb "github.com/youtube/vitess/go/vt/proto/topodata" ) @@ -152,22 +154,30 @@ func (ki *KeyspaceInfo) ComputeCellServedFrom(cell string) map[TabletType]string } // UpdateKeyspace updates the keyspace data, with the right version -func UpdateKeyspace(ctx context.Context, ts Server, ki *KeyspaceInfo) error { +func (ts Server) UpdateKeyspace(ctx context.Context, ki *KeyspaceInfo) error { var version int64 = -1 if ki.version != 0 { version = ki.version } - newVersion, err := ts.UpdateKeyspace(ctx, ki, version) - if err == nil { - ki.version = newVersion + newVersion, err := ts.Impl.UpdateKeyspace(ctx, ki.keyspace, ki.Keyspace, version) + if err != nil { + return err } - return err + ki.version = newVersion + + event.Dispatch(&events.KeyspaceChange{ + KeyspaceName: ki.keyspace, + Keyspace: ki.Keyspace, + Status: "updated", + }) + + return nil } // FindAllShardsInKeyspace reads and returns all the existing shards in // a keyspace. It doesn't take any lock. -func FindAllShardsInKeyspace(ctx context.Context, ts Server, keyspace string) (map[string]*ShardInfo, error) { +func (ts Server) FindAllShardsInKeyspace(ctx context.Context, keyspace string) (map[string]*ShardInfo, error) { shards, err := ts.GetShardNames(ctx, keyspace) if err != nil { return nil, err @@ -197,3 +207,17 @@ func FindAllShardsInKeyspace(ctx context.Context, ts Server, keyspace string) (m } return result, nil } + +// DeleteKeyspaceShards wraps the underlying Impl.DeleteKeyspaceShards +// and dispatches the event. +func (ts Server) DeleteKeyspaceShards(ctx context.Context, keyspace string) error { + if err := ts.Impl.DeleteKeyspaceShards(ctx, keyspace); err != nil { + return err + } + event.Dispatch(&events.KeyspaceChange{ + KeyspaceName: keyspace, + Keyspace: nil, + Status: "deleted all shards", + }) + return nil +} diff --git a/go/vt/topo/server.go b/go/vt/topo/server.go index c98cc94db9..0b7d2aaad7 100644 --- a/go/vt/topo/server.go +++ b/go/vt/topo/server.go @@ -81,7 +81,7 @@ type Impl interface { // or ErrBadVersion if the version has changed. // // Do not use directly, but instead use topo.UpdateKeyspace. - UpdateKeyspace(ctx context.Context, ki *KeyspaceInfo, existingVersion int64) (newVersion int64, err error) + UpdateKeyspace(ctx context.Context, keyspace string, value *pb.Keyspace, existingVersion int64) (newVersion int64, err error) // DeleteKeyspace deletes the specified keyspace. // Can return ErrNoNode if the keyspace doesn't exist. diff --git a/go/vt/topo/shard.go b/go/vt/topo/shard.go index 25916cc149..c0ea53747d 100644 --- a/go/vt/topo/shard.go +++ b/go/vt/topo/shard.go @@ -264,7 +264,7 @@ func CreateShard(ctx context.Context, ts Server, keyspace, shard string) error { if IsShardUsingRangeBasedSharding(name) { // if we are using range-based sharding, we don't want // overlapping shards to all serve and confuse the clients. - sis, err := FindAllShardsInKeyspace(ctx, ts, keyspace) + sis, err := ts.FindAllShardsInKeyspace(ctx, keyspace) if err != nil && err != ErrNoNode { return err } diff --git a/go/vt/topo/test/faketopo/faketopo.go b/go/vt/topo/test/faketopo/faketopo.go index 0493a8a8d5..d7fa9526d0 100644 --- a/go/vt/topo/test/faketopo/faketopo.go +++ b/go/vt/topo/test/faketopo/faketopo.go @@ -45,7 +45,7 @@ func (ft FakeTopo) CreateKeyspace(ctx context.Context, keyspace string, value *p } // UpdateKeyspace implements topo.Server. -func (ft FakeTopo) UpdateKeyspace(ctx context.Context, ki *topo.KeyspaceInfo, existingVersion int64) (int64, error) { +func (ft FakeTopo) UpdateKeyspace(ctx context.Context, keyspace string, value *pb.Keyspace, existingVersion int64) (int64, error) { return 0, errNotImplemented } diff --git a/go/vt/topo/test/keyspace.go b/go/vt/topo/test/keyspace.go index 3522969d58..89b8841217 100644 --- a/go/vt/topo/test/keyspace.go +++ b/go/vt/topo/test/keyspace.go @@ -103,7 +103,7 @@ func CheckKeyspace(ctx context.Context, t *testing.T, ts topo.Impl) { newServedFroms = append(newServedFroms, ksf) } ki.ServedFroms = newServedFroms - err = topo.UpdateKeyspace(ctx, topo.Server{Impl: ts}, ki) + _, err = ts.UpdateKeyspace(ctx, ki.KeyspaceName(), ki.Keyspace, ki.Version()) if err != nil { t.Fatalf("UpdateKeyspace: %v", err) } diff --git a/go/vt/topotools/split.go b/go/vt/topotools/split.go index 8764a4bb9c..5acdb4c267 100644 --- a/go/vt/topotools/split.go +++ b/go/vt/topotools/split.go @@ -54,7 +54,7 @@ func OverlappingShardsForShard(os []*OverlappingShards, shardName string) *Overl // will return an error). // If shards don't perfectly overlap, they are not returned. func FindOverlappingShards(ctx context.Context, ts topo.Server, keyspace string) ([]*OverlappingShards, error) { - shardMap, err := topo.FindAllShardsInKeyspace(ctx, ts, keyspace) + shardMap, err := ts.FindAllShardsInKeyspace(ctx, keyspace) if err != nil { return nil, err } diff --git a/go/vt/vtctl/vtctl.go b/go/vt/vtctl/vtctl.go index c6cc60b1e6..f130820ffc 100644 --- a/go/vt/vtctl/vtctl.go +++ b/go/vt/vtctl/vtctl.go @@ -1716,7 +1716,7 @@ func commandFindAllShardsInKeyspace(ctx context.Context, wr *wrangler.Wrangler, } keyspace := subFlags.Arg(0) - result, err := topo.FindAllShardsInKeyspace(ctx, wr.TopoServer(), keyspace) + result, err := wr.TopoServer().FindAllShardsInKeyspace(ctx, keyspace) if err != nil { return err } diff --git a/go/vt/wrangler/keyspace.go b/go/vt/wrangler/keyspace.go index 188feef097..9cdd5dc126 100644 --- a/go/vt/wrangler/keyspace.go +++ b/go/vt/wrangler/keyspace.go @@ -74,7 +74,7 @@ func (wr *Wrangler) setKeyspaceShardingInfo(ctx context.Context, keyspace, shard ki.ShardingColumnName = shardingColumnName ki.ShardingColumnType = shardingColumnType ki.SplitShardCount = splitShardCount - return topo.UpdateKeyspace(ctx, wr.ts, ki) + return wr.ts.UpdateKeyspace(ctx, ki) } // MigrateServedTypes is used during horizontal splits to migrate a @@ -576,7 +576,7 @@ func (wr *Wrangler) migrateServedFrom(ctx context.Context, ki *topo.KeyspaceInfo func (wr *Wrangler) replicaMigrateServedFrom(ctx context.Context, ki *topo.KeyspaceInfo, sourceShard *topo.ShardInfo, destinationShard *topo.ShardInfo, servedType pb.TabletType, cells []string, reverse bool, tables []string, ev *events.MigrateServedFrom) error { // Save the destination keyspace (its ServedFrom has been changed) event.DispatchUpdate(ev, "updating keyspace") - if err := topo.UpdateKeyspace(ctx, wr.ts, ki); err != nil { + if err := wr.ts.UpdateKeyspace(ctx, ki); err != nil { return err } @@ -653,7 +653,7 @@ func (wr *Wrangler) masterMigrateServedFrom(ctx context.Context, ki *topo.Keyspa // Update the destination keyspace (its ServedFrom has changed) event.DispatchUpdate(ev, "updating keyspace") - if err = topo.UpdateKeyspace(ctx, wr.ts, ki); err != nil { + if err = wr.ts.UpdateKeyspace(ctx, ki); err != nil { return err } @@ -695,7 +695,7 @@ func (wr *Wrangler) setKeyspaceServedFrom(ctx context.Context, keyspace string, if err := ki.UpdateServedFromMap(servedType, cells, sourceKeyspace, remove, nil); err != nil { return err } - return topo.UpdateKeyspace(ctx, wr.ts, ki) + return wr.ts.UpdateKeyspace(ctx, ki) } // RefreshTablesByShard calls RefreshState on all the tables of a diff --git a/go/vt/wrangler/rebuild.go b/go/vt/wrangler/rebuild.go index c023553bb0..66537add92 100644 --- a/go/vt/wrangler/rebuild.go +++ b/go/vt/wrangler/rebuild.go @@ -101,7 +101,7 @@ func (wr *Wrangler) rebuildKeyspace(ctx context.Context, keyspace string, cells } } else { - shardCache, err = topo.FindAllShardsInKeyspace(ctx, wr.ts, keyspace) + shardCache, err = wr.ts.FindAllShardsInKeyspace(ctx, keyspace) if err != nil { return err } @@ -118,7 +118,7 @@ func (wr *Wrangler) rebuildKeyspace(ctx context.Context, keyspace string, cells // Then we add the cells from the keyspaces we might be 'ServedFrom'. for _, ksf := range ki.ServedFroms { - servedFromShards, err := topo.FindAllShardsInKeyspace(ctx, wr.ts, ksf.Keyspace) + servedFromShards, err := wr.ts.FindAllShardsInKeyspace(ctx, ksf.Keyspace) if err != nil { return err } diff --git a/go/vt/zktopo/keyspace.go b/go/vt/zktopo/keyspace.go index 1c3908df19..4956290a77 100644 --- a/go/vt/zktopo/keyspace.go +++ b/go/vt/zktopo/keyspace.go @@ -71,9 +71,9 @@ func (zkts *Server) CreateKeyspace(ctx context.Context, keyspace string, value * } // UpdateKeyspace is part of the topo.Server interface -func (zkts *Server) UpdateKeyspace(ctx context.Context, ki *topo.KeyspaceInfo, existingVersion int64) (int64, error) { - keyspacePath := path.Join(globalKeyspacesPath, ki.KeyspaceName()) - data, err := json.MarshalIndent(ki.Keyspace, "", " ") +func (zkts *Server) UpdateKeyspace(ctx context.Context, keyspace string, value *pb.Keyspace, existingVersion int64) (int64, error) { + keyspacePath := path.Join(globalKeyspacesPath, keyspace) + data, err := json.MarshalIndent(value, "", " ") if err != nil { return -1, err } @@ -85,11 +85,6 @@ func (zkts *Server) UpdateKeyspace(ctx context.Context, ki *topo.KeyspaceInfo, e return -1, err } - event.Dispatch(&events.KeyspaceChange{ - KeyspaceName: ki.KeyspaceName(), - Keyspace: ki.Keyspace, - Status: "updated", - }) return int64(stat.Version()), nil } @@ -151,11 +146,5 @@ func (zkts *Server) DeleteKeyspaceShards(ctx context.Context, keyspace string) e if err := zk.DeleteRecursive(zkts.zconn, shardsPath, -1); err != nil && !zookeeper.IsError(err, zookeeper.ZNONODE) { return err } - - event.Dispatch(&events.KeyspaceChange{ - KeyspaceName: keyspace, - Keyspace: nil, - Status: "deleted all shards", - }) return nil } From 744d845d83c9917ce5a6f0114190cf8e698d5c44 Mon Sep 17 00:00:00 2001 From: Alain Jobart Date: Wed, 19 Aug 2015 18:47:42 -0700 Subject: [PATCH 63/80] Simplifying zktopo.NewServer --- go/cmd/vtgate/plugin_fakezk.go | 2 +- go/vt/topo/helpers/copy_test.go | 4 ++-- go/vt/topo/helpers/tee_topo_test.go | 4 ++-- go/vt/zktopo/server.go | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/go/cmd/vtgate/plugin_fakezk.go b/go/cmd/vtgate/plugin_fakezk.go index f38325961b..5be7c7d463 100644 --- a/go/cmd/vtgate/plugin_fakezk.go +++ b/go/cmd/vtgate/plugin_fakezk.go @@ -18,6 +18,6 @@ var fakezkConfig = flag.String("fakezk-config", "", "If set, will read the json func init() { if *fakezkConfig != "" { - topo.RegisterServer("fakezk", zktopo.NewServer(fakezk.NewConnFromFile(*fakezkConfig)).Impl) + topo.RegisterServer("fakezk", zktopo.NewServer(fakezk.NewConnFromFile(*fakezkConfig))) } } diff --git a/go/vt/topo/helpers/copy_test.go b/go/vt/topo/helpers/copy_test.go index c01e373822..7196687f59 100644 --- a/go/vt/topo/helpers/copy_test.go +++ b/go/vt/topo/helpers/copy_test.go @@ -22,10 +22,10 @@ import ( func createSetup(ctx context.Context, t *testing.T) (topo.Impl, topo.Impl) { fromConn := fakezk.NewConn() - fromTS := zktopo.NewServer(fromConn).Impl + fromTS := zktopo.NewServer(fromConn) toConn := fakezk.NewConn() - toTS := zktopo.NewServer(toConn).Impl + toTS := zktopo.NewServer(toConn) for _, zkPath := range []string{"/zk/test_cell/vt", "/zk/global/vt"} { if _, err := zk.CreateRecursive(fromConn, zkPath, "", 0, zookeeper.WorldACL(zookeeper.PERM_ALL)); err != nil { diff --git a/go/vt/topo/helpers/tee_topo_test.go b/go/vt/topo/helpers/tee_topo_test.go index ca00966440..bd4e17ec49 100644 --- a/go/vt/topo/helpers/tee_topo_test.go +++ b/go/vt/topo/helpers/tee_topo_test.go @@ -42,8 +42,8 @@ func newFakeTeeServer(t *testing.T) topo.Impl { t.Fatalf("cannot init ZooKeeper: %v", err) } } - s1 := fakeServer{Impl: zktopo.NewServer(zconn1).Impl, localCells: cells[:len(cells)-1]} - s2 := fakeServer{Impl: zktopo.NewServer(zconn2).Impl, localCells: cells[:len(cells)-1]} + s1 := fakeServer{Impl: zktopo.NewServer(zconn1), localCells: cells[:len(cells)-1]} + s2 := fakeServer{Impl: zktopo.NewServer(zconn2), localCells: cells[:len(cells)-1]} return NewTee(s1, s2, false) } diff --git a/go/vt/zktopo/server.go b/go/vt/zktopo/server.go index bf45cdd1a2..6237f6afe3 100644 --- a/go/vt/zktopo/server.go +++ b/go/vt/zktopo/server.go @@ -33,8 +33,8 @@ func (zkts *Server) GetZConn() zk.Conn { // NewServer can be used to create a custom Server // (for tests for instance) but it cannot change the globally // registered one. -func NewServer(zconn zk.Conn) topo.Server { - return topo.Server{Impl: &Server{zconn: zconn}} +func NewServer(zconn zk.Conn) topo.Impl { + return &Server{zconn: zconn} } func init() { From 65bb366c8e8ff266e44cdf4c0f48376000a2ebd4 Mon Sep 17 00:00:00 2001 From: Alain Jobart Date: Wed, 19 Aug 2015 18:50:57 -0700 Subject: [PATCH 64/80] Simplifying these tests too. --- go/vt/etcdtopo/explorer_test.go | 8 ++++---- go/vt/etcdtopo/server_test.go | 3 +-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/go/vt/etcdtopo/explorer_test.go b/go/vt/etcdtopo/explorer_test.go index 71ba35becf..fa0b2b2fd4 100644 --- a/go/vt/etcdtopo/explorer_test.go +++ b/go/vt/etcdtopo/explorer_test.go @@ -73,7 +73,7 @@ func TestHandlePathRoot(t *testing.T) { want := []string{"global", "cell1", "cell2", "cell3"} ts := newTestServer(t, cells) - ex := NewExplorer(ts.(*Server)) + ex := NewExplorer(ts) result := ex.HandlePath(nil, input, nil) exResult := result.(*explorerResult) if got := exResult.Children; !reflect.DeepEqual(got, want) { @@ -101,7 +101,7 @@ func TestHandlePathKeyspace(t *testing.T) { } m := &mockActionRepo{} - ex := NewExplorer(ts.(*Server)) + ex := NewExplorer(ts) result := ex.HandlePath(m, input, nil) exResult := result.(*explorerResult) if got := exResult.Data; got != want { @@ -135,7 +135,7 @@ func TestHandlePathShard(t *testing.T) { } m := &mockActionRepo{} - ex := NewExplorer(ts.(*Server)) + ex := NewExplorer(ts) result := ex.HandlePath(m, input, nil) exResult := result.(*explorerResult) if got := exResult.Data; got != want { @@ -169,7 +169,7 @@ func TestHandlePathTablet(t *testing.T) { } m := &mockActionRepo{} - ex := NewExplorer(ts.(*Server)) + ex := NewExplorer(ts) result := ex.HandlePath(m, input, nil) exResult := result.(*explorerResult) if got := exResult.Data; got != want { diff --git a/go/vt/etcdtopo/server_test.go b/go/vt/etcdtopo/server_test.go index b80e42333b..0075bc6b81 100644 --- a/go/vt/etcdtopo/server_test.go +++ b/go/vt/etcdtopo/server_test.go @@ -8,12 +8,11 @@ import ( "testing" "github.com/youtube/vitess/go/flagutil" - "github.com/youtube/vitess/go/vt/topo" "github.com/youtube/vitess/go/vt/topo/test" "golang.org/x/net/context" ) -func newTestServer(t *testing.T, cells []string) topo.Impl { +func newTestServer(t *testing.T, cells []string) *Server { s := &Server{ _cells: make(map[string]*cellClient), newClient: newTestClient, From 92e3640a1be4b2d800a712f490546ce2c1288e96 Mon Sep 17 00:00:00 2001 From: Alain Jobart Date: Wed, 19 Aug 2015 19:03:33 -0700 Subject: [PATCH 65/80] Simplifying zkto test server a bit. --- go/vt/topo/server.go | 7 ------- go/vt/zktopo/testserver.go | 11 ++++++++--- go/vt/zktopo/zktopo_test.go | 24 ++++++++++++------------ 3 files changed, 20 insertions(+), 22 deletions(-) diff --git a/go/vt/topo/server.go b/go/vt/topo/server.go index 0b7d2aaad7..358d0ce84e 100644 --- a/go/vt/topo/server.go +++ b/go/vt/topo/server.go @@ -325,13 +325,6 @@ type Server struct { Impl } -// Schemafier is a temporary interface for supporting vschema -// reads and writes. It will eventually be merged into Server. -//type Schemafier interface { -// SaveVSchema(context.Context, string) error -// GetVSchema(ctx context.Context) (string, error) -//} - // Registry for Server implementations. var serverImpls = make(map[string]Impl) diff --git a/go/vt/zktopo/testserver.go b/go/vt/zktopo/testserver.go index 7dff1b835d..fa1d570276 100644 --- a/go/vt/zktopo/testserver.go +++ b/go/vt/zktopo/testserver.go @@ -20,8 +20,8 @@ type TestServer struct { HookLockSrvShardForAction func() } -// NewTestServer returns a new TestServer (with the required paths created) -func NewTestServer(t *testing.T, cells []string) topo.Server { +// newTestServer returns a new TestServer (with the required paths created) +func newTestServer(t *testing.T, cells []string) topo.Impl { zconn := fakezk.NewConn() // create the toplevel zk paths @@ -33,7 +33,12 @@ func NewTestServer(t *testing.T, cells []string) topo.Server { t.Fatalf("cannot init ZooKeeper: %v", err) } } - return topo.Server{Impl: &TestServer{Impl: &Server{zconn}, localCells: cells}} + return &TestServer{Impl: &Server{zconn}, localCells: cells} +} + +// NewTestServer returns a new TestServer (with the required paths created) +func NewTestServer(t *testing.T, cells []string) topo.Server { + return topo.Server{Impl: newTestServer(t, cells)} } // GetKnownCells is part of topo.Server interface diff --git a/go/vt/zktopo/zktopo_test.go b/go/vt/zktopo/zktopo_test.go index d581445c87..2b592184f1 100644 --- a/go/vt/zktopo/zktopo_test.go +++ b/go/vt/zktopo/zktopo_test.go @@ -16,48 +16,48 @@ import ( func TestKeyspace(t *testing.T) { ctx := context.Background() - ts := NewTestServer(t, []string{"test"}).Impl + ts := newTestServer(t, []string{"test"}) defer ts.Close() test.CheckKeyspace(ctx, t, ts) } func TestShard(t *testing.T) { ctx := context.Background() - ts := NewTestServer(t, []string{"test"}).Impl + ts := newTestServer(t, []string{"test"}) defer ts.Close() test.CheckShard(ctx, t, ts) } func TestTablet(t *testing.T) { ctx := context.Background() - ts := NewTestServer(t, []string{"test"}).Impl + ts := newTestServer(t, []string{"test"}) defer ts.Close() test.CheckTablet(ctx, t, ts) } func TestShardReplication(t *testing.T) { ctx := context.Background() - ts := NewTestServer(t, []string{"test"}).Impl + ts := newTestServer(t, []string{"test"}) defer ts.Close() test.CheckShardReplication(ctx, t, ts) } func TestServingGraph(t *testing.T) { - ts := NewTestServer(t, []string{"test"}).Impl + ts := newTestServer(t, []string{"test"}) defer ts.Close() test.CheckServingGraph(context.Background(), t, ts) } func TestWatchSrvKeyspace(t *testing.T) { WatchSleepDuration = 2 * time.Millisecond - ts := NewTestServer(t, []string{"test"}).Impl + ts := newTestServer(t, []string{"test"}) defer ts.Close() test.CheckWatchSrvKeyspace(context.Background(), t, ts) } func TestKeyspaceLock(t *testing.T) { ctx := context.Background() - ts := NewTestServer(t, []string{"test"}).Impl + ts := newTestServer(t, []string{"test"}) defer ts.Close() test.CheckKeyspaceLock(ctx, t, ts) } @@ -68,7 +68,7 @@ func TestShardLock(t *testing.T) { t.Skip("skipping wait-based test in short mode.") } - ts := NewTestServer(t, []string{"test"}).Impl + ts := newTestServer(t, []string{"test"}) defer ts.Close() test.CheckShardLock(ctx, t, ts) } @@ -79,14 +79,14 @@ func TestSrvShardLock(t *testing.T) { t.Skip("skipping wait-based test in short mode.") } - ts := NewTestServer(t, []string{"test"}).Impl + ts := newTestServer(t, []string{"test"}) defer ts.Close() test.CheckSrvShardLock(ctx, t, ts) } func TestVSchema(t *testing.T) { ctx := context.Background() - ts := NewTestServer(t, []string{"test"}).Impl + ts := newTestServer(t, []string{"test"}) defer ts.Close() test.CheckVSchema(ctx, t, ts) } @@ -94,7 +94,7 @@ func TestVSchema(t *testing.T) { // TestPurgeActions is a ZK specific unit test func TestPurgeActions(t *testing.T) { ctx := context.Background() - ts := NewTestServer(t, []string{"test"}).Impl + ts := newTestServer(t, []string{"test"}) defer ts.Close() if err := ts.CreateKeyspace(ctx, "test_keyspace", &pb.Keyspace{}); err != nil { @@ -126,7 +126,7 @@ func TestPurgeActions(t *testing.T) { // TestPruneActionLogs is a ZK specific unit test func TestPruneActionLogs(t *testing.T) { ctx := context.Background() - ts := NewTestServer(t, []string{"test"}).Impl + ts := newTestServer(t, []string{"test"}) defer ts.Close() if err := ts.CreateKeyspace(ctx, "test_keyspace", &pb.Keyspace{}); err != nil { From 6bf93194a5f1c16e6e07f7f52a0d71b8ae1b9fca Mon Sep 17 00:00:00 2001 From: Shengzhe Yao Date: Wed, 19 Aug 2015 22:38:20 -0700 Subject: [PATCH 66/80] SchemaManager should be able to handle nil ShardInfo.MasterAlias. ShardInfo.GetShard may contain nil MasterAlias because when creating a shard, or when scrapping the shard's master, the MasterAlias will be nil. SchemaManager should return error in such case instead of throwing a panic. --- go/vt/schemamanager/schemamanager_test.go | 11 ++++++++--- go/vt/schemamanager/tablet_executor.go | 4 ++++ go/vt/schemamanager/tablet_executor_test.go | 14 ++++++++++++++ 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/go/vt/schemamanager/schemamanager_test.go b/go/vt/schemamanager/schemamanager_test.go index 4dac2ae44b..4a28d1f653 100644 --- a/go/vt/schemamanager/schemamanager_test.go +++ b/go/vt/schemamanager/schemamanager_test.go @@ -269,6 +269,7 @@ func (client *fakeTabletManagerClient) ExecuteFetchAsDba(ctx context.Context, ta type fakeTopo struct { faketopo.FakeTopo + WithEmptyMasterAlias bool } func newFakeTopo() *fakeTopo { @@ -284,11 +285,15 @@ func (topoServer *fakeTopo) GetShardNames(ctx context.Context, keyspace string) } func (topoServer *fakeTopo) GetShard(ctx context.Context, keyspace string, shard string) (*topo.ShardInfo, error) { - value := &pb.Shard{ - MasterAlias: &pb.TabletAlias{ + var masterAlias *pb.TabletAlias + if !topoServer.WithEmptyMasterAlias { + masterAlias = &pb.TabletAlias{ Cell: "test_cell", Uid: 0, - }, + } + } + value := &pb.Shard{ + MasterAlias: masterAlias, } return topo.NewShardInfo(keyspace, shard, value, 0), nil } diff --git a/go/vt/schemamanager/tablet_executor.go b/go/vt/schemamanager/tablet_executor.go index 7eb2412d99..4b3cb69f73 100644 --- a/go/vt/schemamanager/tablet_executor.go +++ b/go/vt/schemamanager/tablet_executor.go @@ -54,6 +54,10 @@ func (exec *TabletExecutor) Open(ctx context.Context, keyspace string) error { if err != nil { return fmt.Errorf("unable to get shard info, keyspace: %s, shard: %s, error: %v", keyspace, shardName, err) } + if !shardInfo.HasMaster() { + log.Errorf("shard: %s does not have a master", shardName) + return fmt.Errorf("shard: %s does not have a master", shardName) + } tabletInfo, err := exec.topoServer.GetTablet(ctx, shardInfo.MasterAlias) if err != nil { return fmt.Errorf("unable to get master tablet info, keyspace: %s, shard: %s, error: %v", keyspace, shardName, err) diff --git a/go/vt/schemamanager/tablet_executor_test.go b/go/vt/schemamanager/tablet_executor_test.go index 7f3b3903d2..f75c1b79b0 100644 --- a/go/vt/schemamanager/tablet_executor_test.go +++ b/go/vt/schemamanager/tablet_executor_test.go @@ -26,6 +26,20 @@ func TestTabletExecutorOpen(t *testing.T) { } } +func TestTabletExecutorOpenWithEmptyMasterAlias(t *testing.T) { + fakeTopo := newFakeTopo() + fakeTopo.WithEmptyMasterAlias = true + executor := NewTabletExecutor( + newFakeTabletManagerClient(), + fakeTopo) + ctx := context.Background() + + if err := executor.Open(ctx, "test_keyspace"); err == nil { + t.Fatalf("executor.Open() = nil, want error") + } + executor.Close() +} + func TestTabletExecutorValidate(t *testing.T) { fakeTmc := newFakeTabletManagerClient() From dcdc993b154ab552c1e7197b5702d1d76403f92e Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Thu, 20 Aug 2015 00:34:53 -0700 Subject: [PATCH 67/80] sqlparser: improved error message Put quotes around the last token to make the error message more readable. --- data/test/sqlparser_test/parse_fail.sql | 32 ++++++++++++------------- data/test/tabletserver/exec_cases.txt | 2 +- data/test/tabletserver/stream_cases.txt | 2 +- data/test/vtgate/select_cases.txt | 2 +- go/vt/sqlparser/token.go | 2 +- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/data/test/sqlparser_test/parse_fail.sql b/data/test/sqlparser_test/parse_fail.sql index bf9c1341ef..631f3ed354 100644 --- a/data/test/sqlparser_test/parse_fail.sql +++ b/data/test/sqlparser_test/parse_fail.sql @@ -1,18 +1,18 @@ -select !8 from t#syntax error at position 9 near ! -select $ from t#syntax error at position 9 near $ -select : from t#syntax error at position 9 near : -select 078 from t#syntax error at position 11 near 078 -select `1a` from t#syntax error at position 9 near 1 -select `:table` from t#syntax error at position 9 near : -select `table:` from t#syntax error at position 14 near table -select 'aa\#syntax error at position 12 near aa -select 'aa#syntax error at position 12 near aa -select * from t where :1 = 2#syntax error at position 24 near : -select * from t where :. = 2#syntax error at position 24 near : -select * from t where ::1 = 2#syntax error at position 25 near :: -select * from t where ::. = 2#syntax error at position 25 near :: -update a set c = values(1)#syntax error at position 24 near values -update a set c = last_insert_id(1)#syntax error at position 32 near last_insert_id +select !8 from t#syntax error at position 9 near '!' +select $ from t#syntax error at position 9 near '$' +select : from t#syntax error at position 9 near ':' +select 078 from t#syntax error at position 11 near '078' +select `1a` from t#syntax error at position 9 near '1' +select `:table` from t#syntax error at position 9 near ':' +select `table:` from t#syntax error at position 14 near 'table' +select 'aa\#syntax error at position 12 near 'aa' +select 'aa#syntax error at position 12 near 'aa' +select * from t where :1 = 2#syntax error at position 24 near ':' +select * from t where :. = 2#syntax error at position 24 near ':' +select * from t where ::1 = 2#syntax error at position 25 near '::' +select * from t where ::. = 2#syntax error at position 25 near '::' +update a set c = values(1)#syntax error at position 24 near 'values' +update a set c = last_insert_id(1)#syntax error at position 32 near 'last_insert_id' select(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(#max nesting level reached at position 406 select(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(F(#syntax error at position 405 -select /* aa#syntax error at position 13 near /* aa +select /* aa#syntax error at position 13 near '/* aa' diff --git a/data/test/tabletserver/exec_cases.txt b/data/test/tabletserver/exec_cases.txt index 035271a6c3..0e738e3144 100644 --- a/data/test/tabletserver/exec_cases.txt +++ b/data/test/tabletserver/exec_cases.txt @@ -2629,4 +2629,4 @@ # syntax error "syntax error" -"syntax error at position 7 near syntax" +"syntax error at position 7 near 'syntax'" diff --git a/data/test/tabletserver/stream_cases.txt b/data/test/tabletserver/stream_cases.txt index d61dacd9ef..b51d11505f 100644 --- a/data/test/tabletserver/stream_cases.txt +++ b/data/test/tabletserver/stream_cases.txt @@ -71,4 +71,4 @@ # syntax error "syntax error" -"syntax error at position 7 near syntax" +"syntax error at position 7 near 'syntax'" diff --git a/data/test/vtgate/select_cases.txt b/data/test/vtgate/select_cases.txt index 22ef1f0179..d1f8fa9cad 100644 --- a/data/test/vtgate/select_cases.txt +++ b/data/test/vtgate/select_cases.txt @@ -2,7 +2,7 @@ "the quick brown fox" { "ID":"NoPlan", - "Reason":"syntax error at position 4 near the", + "Reason":"syntax error at position 4 near 'the'", "Table": "", "Original":"the quick brown fox", "Rewritten":"", diff --git a/go/vt/sqlparser/token.go b/go/vt/sqlparser/token.go index a73dc5912a..ff82b55e4f 100644 --- a/go/vt/sqlparser/token.go +++ b/go/vt/sqlparser/token.go @@ -131,7 +131,7 @@ func (tkn *Tokenizer) Lex(lval *yySymType) int { func (tkn *Tokenizer) Error(err string) { buf := bytes.NewBuffer(make([]byte, 0, 32)) if tkn.lastToken != nil { - fmt.Fprintf(buf, "%s at position %v near %s", err, tkn.Position, tkn.lastToken) + fmt.Fprintf(buf, "%s at position %v near '%s'", err, tkn.Position, tkn.lastToken) } else { fmt.Fprintf(buf, "%s at position %v", err, tkn.Position) } From 9a3fa8204d9db7d236f2c3ece4d5b7e13af13578 Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Thu, 20 Aug 2015 01:06:47 -0700 Subject: [PATCH 68/80] tabletserver: terse error change If the original request doesn't have bind variables, there's no point resorting to terse errors. Might as well return the full mysql error. --- go/vt/tabletserver/sqlquery.go | 2 +- go/vt/tabletserver/sqlquery_test.go | 28 +++++++++++++++++++++++++++- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/go/vt/tabletserver/sqlquery.go b/go/vt/tabletserver/sqlquery.go index adf501b73d..cbdff010c3 100644 --- a/go/vt/tabletserver/sqlquery.go +++ b/go/vt/tabletserver/sqlquery.go @@ -369,7 +369,7 @@ func (sq *SqlQuery) handleExecErrorNoPanic(query *proto.Query, err interface{}, return NewTabletError(ErrFail, "%v: uncaught panic for %v", err, query) } var myError error - if sq.config.TerseErrors && terr.SqlError != 0 { + if sq.config.TerseErrors && terr.SqlError != 0 && len(query.BindVariables) != 0 { myError = fmt.Errorf("%s(errno %d) during query: %s", terr.Prefix(), terr.SqlError, query.Sql) } else { myError = terr diff --git a/go/vt/tabletserver/sqlquery_test.go b/go/vt/tabletserver/sqlquery_test.go index 9b43dc3aa5..c6e20d1633 100644 --- a/go/vt/tabletserver/sqlquery_test.go +++ b/go/vt/tabletserver/sqlquery_test.go @@ -1034,7 +1034,7 @@ func TestTerseErrors2(t *testing.T) { logStats := newSqlQueryStats("TestHandleExecError", ctx) query := proto.Query{ Sql: "select * from test_table", - BindVariables: nil, + BindVariables: map[string]interface{}{"a": 1}, } var err error defer func() { @@ -1055,6 +1055,32 @@ func TestTerseErrors2(t *testing.T) { }) } +func TestTerseErrors3(t *testing.T) { + ctx := context.Background() + logStats := newSqlQueryStats("TestHandleExecError", ctx) + query := proto.Query{ + Sql: "select * from test_table", + BindVariables: nil, + } + var err error + defer func() { + want := "error: msg" + if err == nil || err.Error() != want { + t.Errorf("Error: %v, want '%s'", err, want) + } + }() + testUtils := newTestUtils() + config := testUtils.newQueryServiceConfig() + sqlQuery := NewSqlQuery(config) + sqlQuery.config.TerseErrors = true + defer sqlQuery.handleExecError(&query, &err, logStats) + panic(&TabletError{ + ErrorType: ErrFail, + Message: "msg", + SqlError: 10, + }) +} + func setUpSQLQueryTest() *fakesqldb.DB { db := fakesqldb.Register() for query, result := range getSupportedQueries() { From 339a1f8bf909863f4da85fe523dbb06d4591a144 Mon Sep 17 00:00:00 2001 From: Alain Jobart Date: Thu, 20 Aug 2015 07:47:15 -0700 Subject: [PATCH 69/80] Finishing up the keyspace part of the topo server reorg. --- go/vt/etcdtopo/keyspace.go | 24 ++------ go/vt/schemamanager/tablet_executor_test.go | 6 +- go/vt/topo/helpers/copy.go | 4 +- go/vt/topo/helpers/tee.go | 20 +++---- go/vt/topo/keyspace.go | 58 ++++++++++++++----- go/vt/topo/keyspace_test.go | 56 ++++++++++-------- go/vt/topo/server.go | 4 +- go/vt/topo/test/faketopo/faketopo.go | 4 +- go/vt/topo/test/keyspace.go | 28 ++++----- go/vt/topotools/events/migrate.go | 4 +- go/vt/topotools/events/migrate_syslog.go | 4 +- go/vt/topotools/events/migrate_syslog_test.go | 8 +-- go/vt/wrangler/keyspace.go | 4 +- go/vt/zktopo/keyspace.go | 22 ++----- 14 files changed, 127 insertions(+), 119 deletions(-) diff --git a/go/vt/etcdtopo/keyspace.go b/go/vt/etcdtopo/keyspace.go index 307f7e374c..58acfe3722 100644 --- a/go/vt/etcdtopo/keyspace.go +++ b/go/vt/etcdtopo/keyspace.go @@ -9,10 +9,8 @@ import ( "fmt" "sync" - "github.com/youtube/vitess/go/event" "github.com/youtube/vitess/go/vt/concurrency" "github.com/youtube/vitess/go/vt/topo" - "github.com/youtube/vitess/go/vt/topo/events" "golang.org/x/net/context" pb "github.com/youtube/vitess/go/vt/proto/topodata" @@ -32,12 +30,6 @@ func (s *Server) CreateKeyspace(ctx context.Context, keyspace string, value *pb. if err = initLockFile(global, keyspaceDirPath(keyspace)); err != nil { return err } - - event.Dispatch(&events.KeyspaceChange{ - KeyspaceName: keyspace, - Keyspace: value, - Status: "created", - }) return nil } @@ -60,21 +52,21 @@ func (s *Server) UpdateKeyspace(ctx context.Context, keyspace string, value *pb. } // GetKeyspace implements topo.Server. -func (s *Server) GetKeyspace(ctx context.Context, keyspace string) (*topo.KeyspaceInfo, error) { +func (s *Server) GetKeyspace(ctx context.Context, keyspace string) (*pb.Keyspace, int64, error) { resp, err := s.getGlobal().Get(keyspaceFilePath(keyspace), false /* sort */, false /* recursive */) if err != nil { - return nil, convertError(err) + return nil, 0, convertError(err) } if resp.Node == nil { - return nil, ErrBadResponse + return nil, 0, ErrBadResponse } value := &pb.Keyspace{} if err := json.Unmarshal([]byte(resp.Node.Value), value); err != nil { - return nil, fmt.Errorf("bad keyspace data (%v): %q", err, resp.Node.Value) + return nil, 0, fmt.Errorf("bad keyspace data (%v): %q", err, resp.Node.Value) } - return topo.NewKeyspaceInfo(keyspace, value, int64(resp.Node.ModifiedIndex)), nil + return value, int64(resp.Node.ModifiedIndex), nil } // GetKeyspaces implements topo.Server. @@ -122,11 +114,5 @@ func (s *Server) DeleteKeyspace(ctx context.Context, keyspace string) error { if err != nil { return convertError(err) } - - event.Dispatch(&events.KeyspaceChange{ - KeyspaceName: keyspace, - Keyspace: nil, - Status: "deleted", - }) return nil } diff --git a/go/vt/schemamanager/tablet_executor_test.go b/go/vt/schemamanager/tablet_executor_test.go index f75c1b79b0..6f91637af3 100644 --- a/go/vt/schemamanager/tablet_executor_test.go +++ b/go/vt/schemamanager/tablet_executor_test.go @@ -27,11 +27,11 @@ func TestTabletExecutorOpen(t *testing.T) { } func TestTabletExecutorOpenWithEmptyMasterAlias(t *testing.T) { - fakeTopo := newFakeTopo() - fakeTopo.WithEmptyMasterAlias = true + ft := newFakeTopo() + ft.Impl.(*fakeTopo).WithEmptyMasterAlias = true executor := NewTabletExecutor( newFakeTabletManagerClient(), - fakeTopo) + ft) ctx := context.Background() if err := executor.Open(ctx, "test_keyspace"); err == nil { diff --git a/go/vt/topo/helpers/copy.go b/go/vt/topo/helpers/copy.go index 98222fe47a..bb6945fc06 100644 --- a/go/vt/topo/helpers/copy.go +++ b/go/vt/topo/helpers/copy.go @@ -32,13 +32,13 @@ func CopyKeyspaces(ctx context.Context, fromTS, toTS topo.Impl) { go func(keyspace string) { defer wg.Done() - k, err := fromTS.GetKeyspace(ctx, keyspace) + k, _, err := fromTS.GetKeyspace(ctx, keyspace) if err != nil { rec.RecordError(fmt.Errorf("GetKeyspace(%v): %v", keyspace, err)) return } - if err := toTS.CreateKeyspace(ctx, keyspace, k.Keyspace); err != nil { + if err := toTS.CreateKeyspace(ctx, keyspace, k); err != nil { if err == topo.ErrNodeExists { log.Warningf("keyspace %v already exists", keyspace) } else { diff --git a/go/vt/topo/helpers/tee.go b/go/vt/topo/helpers/tee.go index 1710054c6a..c0bcf16056 100644 --- a/go/vt/topo/helpers/tee.go +++ b/go/vt/topo/helpers/tee.go @@ -141,14 +141,14 @@ func (tee *Tee) UpdateKeyspace(ctx context.Context, keyspace string, value *pb.K log.Warningf("secondary.CreateKeyspace(%v) failed (after UpdateKeyspace returned ErrNoNode): %v", keyspace, serr) } else { log.Infof("secondary.UpdateKeyspace(%v) failed with ErrNoNode, CreateKeyspace then worked.", keyspace) - ki, gerr := tee.secondary.GetKeyspace(ctx, keyspace) + _, secondaryVersion, gerr := tee.secondary.GetKeyspace(ctx, keyspace) if gerr != nil { log.Warningf("Failed to re-read keyspace(%v) after creating it on secondary: %v", keyspace, gerr) } else { tee.mu.Lock() tee.keyspaceVersionMapping[keyspace] = versionMapping{ readFromVersion: newVersion, - readFromSecondVersion: ki.Version(), + readFromSecondVersion: secondaryVersion, } tee.mu.Unlock() } @@ -182,25 +182,25 @@ func (tee *Tee) DeleteKeyspace(ctx context.Context, keyspace string) error { } // GetKeyspace is part of the topo.Server interface -func (tee *Tee) GetKeyspace(ctx context.Context, keyspace string) (*topo.KeyspaceInfo, error) { - ki, err := tee.readFrom.GetKeyspace(ctx, keyspace) +func (tee *Tee) GetKeyspace(ctx context.Context, keyspace string) (*pb.Keyspace, int64, error) { + k, version, err := tee.readFrom.GetKeyspace(ctx, keyspace) if err != nil { - return nil, err + return nil, 0, err } - ki2, err := tee.readFromSecond.GetKeyspace(ctx, keyspace) + _, version2, err := tee.readFromSecond.GetKeyspace(ctx, keyspace) if err != nil { // can't read from secondary, so we can's keep version map - return ki, nil + return k, version, nil } tee.mu.Lock() tee.keyspaceVersionMapping[keyspace] = versionMapping{ - readFromVersion: ki.Version(), - readFromSecondVersion: ki2.Version(), + readFromVersion: version, + readFromSecondVersion: version2, } tee.mu.Unlock() - return ki, nil + return k, version, nil } // GetKeyspaces is part of the topo.Server interface diff --git a/go/vt/topo/keyspace.go b/go/vt/topo/keyspace.go index 3002301826..8eb880f68e 100644 --- a/go/vt/topo/keyspace.go +++ b/go/vt/topo/keyspace.go @@ -34,22 +34,6 @@ func (ki *KeyspaceInfo) KeyspaceName() string { return ki.keyspace } -// Version returns the keyspace version from last time it was read or updated. -func (ki *KeyspaceInfo) Version() int64 { - return ki.version -} - -// NewKeyspaceInfo returns a KeyspaceInfo basing on keyspace with the -// keyspace. This function should be only used by Server -// implementations. -func NewKeyspaceInfo(keyspace string, value *pb.Keyspace, version int64) *KeyspaceInfo { - return &KeyspaceInfo{ - keyspace: keyspace, - version: version, - Keyspace: value, - } -} - // GetServedFrom returns a Keyspace_ServedFrom record if it exists. func (ki *KeyspaceInfo) GetServedFrom(tabletType pb.TabletType) *pb.Keyspace_ServedFrom { for _, ksf := range ki.ServedFroms { @@ -153,6 +137,34 @@ func (ki *KeyspaceInfo) ComputeCellServedFrom(cell string) map[TabletType]string return result } +// CreateKeyspace wraps the underlying Impl.DeleteKeyspaceShards +// and dispatches the event. +func (ts Server) CreateKeyspace(ctx context.Context, keyspace string, value *pb.Keyspace) error { + if err := ts.Impl.CreateKeyspace(ctx, keyspace, value); err != nil { + return err + } + event.Dispatch(&events.KeyspaceChange{ + KeyspaceName: keyspace, + Keyspace: value, + Status: "created", + }) + return nil +} + +// GetKeyspace reads the given keyspace and returns it +func (ts Server) GetKeyspace(ctx context.Context, keyspace string) (*KeyspaceInfo, error) { + value, version, err := ts.Impl.GetKeyspace(ctx, keyspace) + if err != nil { + return nil, err + } + + return &KeyspaceInfo{ + keyspace: keyspace, + version: version, + Keyspace: value, + }, nil +} + // UpdateKeyspace updates the keyspace data, with the right version func (ts Server) UpdateKeyspace(ctx context.Context, ki *KeyspaceInfo) error { var version int64 = -1 @@ -221,3 +233,17 @@ func (ts Server) DeleteKeyspaceShards(ctx context.Context, keyspace string) erro }) return nil } + +// DeleteKeyspace wraps the underlying Impl.DeleteKeyspace +// and dispatches the event. +func (ts Server) DeleteKeyspace(ctx context.Context, keyspace string) error { + if err := ts.Impl.DeleteKeyspace(ctx, keyspace); err != nil { + return err + } + event.Dispatch(&events.KeyspaceChange{ + KeyspaceName: keyspace, + Keyspace: nil, + Status: "deleted", + }) + return nil +} diff --git a/go/vt/topo/keyspace_test.go b/go/vt/topo/keyspace_test.go index d85f63fd6f..b59b37cce2 100644 --- a/go/vt/topo/keyspace_test.go +++ b/go/vt/topo/keyspace_test.go @@ -14,20 +14,24 @@ import ( // This file tests the keyspace related object functionnalities. func TestUpdateServedFromMap(t *testing.T) { - ki := NewKeyspaceInfo("ks", &pb.Keyspace{ - ServedFroms: []*pb.Keyspace_ServedFrom{ - &pb.Keyspace_ServedFrom{ - TabletType: pb.TabletType_RDONLY, - Cells: nil, - Keyspace: "source", - }, - &pb.Keyspace_ServedFrom{ - TabletType: pb.TabletType_MASTER, - Cells: nil, - Keyspace: "source", + ki := &KeyspaceInfo{ + keyspace: "ks", + version: 1, + Keyspace: &pb.Keyspace{ + ServedFroms: []*pb.Keyspace_ServedFrom{ + &pb.Keyspace_ServedFrom{ + TabletType: pb.TabletType_RDONLY, + Cells: nil, + Keyspace: "source", + }, + &pb.Keyspace_ServedFrom{ + TabletType: pb.TabletType_MASTER, + Cells: nil, + Keyspace: "source", + }, }, }, - }, 1) + } allCells := []string{"first", "second", "third"} // migrate one cell @@ -115,20 +119,24 @@ func TestUpdateServedFromMap(t *testing.T) { } func TestComputeCellServedFrom(t *testing.T) { - ki := NewKeyspaceInfo("ks", &pb.Keyspace{ - ServedFroms: []*pb.Keyspace_ServedFrom{ - &pb.Keyspace_ServedFrom{ - TabletType: pb.TabletType_MASTER, - Cells: nil, - Keyspace: "source", - }, - &pb.Keyspace_ServedFrom{ - TabletType: pb.TabletType_REPLICA, - Cells: []string{"c1", "c2"}, - Keyspace: "source", + ki := &KeyspaceInfo{ + keyspace: "ks", + version: 1, + Keyspace: &pb.Keyspace{ + ServedFroms: []*pb.Keyspace_ServedFrom{ + &pb.Keyspace_ServedFrom{ + TabletType: pb.TabletType_MASTER, + Cells: nil, + Keyspace: "source", + }, + &pb.Keyspace_ServedFrom{ + TabletType: pb.TabletType_REPLICA, + Cells: []string{"c1", "c2"}, + Keyspace: "source", + }, }, }, - }, 1) + } m := ki.ComputeCellServedFrom("c3") if !reflect.DeepEqual(m, map[TabletType]string{ diff --git a/go/vt/topo/server.go b/go/vt/topo/server.go index 358d0ce84e..c234a80fb6 100644 --- a/go/vt/topo/server.go +++ b/go/vt/topo/server.go @@ -87,9 +87,9 @@ type Impl interface { // Can return ErrNoNode if the keyspace doesn't exist. DeleteKeyspace(ctx context.Context, keyspace string) error - // GetKeyspace reads a keyspace and returns it. + // GetKeyspace reads a keyspace and returns it, along with its version. // Can return ErrNoNode - GetKeyspace(ctx context.Context, keyspace string) (*KeyspaceInfo, error) + GetKeyspace(ctx context.Context, keyspace string) (*pb.Keyspace, int64, error) // GetKeyspaces returns the known keyspace names. They shall be sorted. GetKeyspaces(ctx context.Context) ([]string, error) diff --git a/go/vt/topo/test/faketopo/faketopo.go b/go/vt/topo/test/faketopo/faketopo.go index d7fa9526d0..b83fdb022d 100644 --- a/go/vt/topo/test/faketopo/faketopo.go +++ b/go/vt/topo/test/faketopo/faketopo.go @@ -55,8 +55,8 @@ func (ft FakeTopo) DeleteKeyspace(ctx context.Context, keyspace string) error { } // GetKeyspace implements topo.Server. -func (ft FakeTopo) GetKeyspace(ctx context.Context, keyspace string) (*topo.KeyspaceInfo, error) { - return nil, errNotImplemented +func (ft FakeTopo) GetKeyspace(ctx context.Context, keyspace string) (*pb.Keyspace, int64, error) { + return nil, 0, errNotImplemented } // GetKeyspaces implements topo.Server. diff --git a/go/vt/topo/test/keyspace.go b/go/vt/topo/test/keyspace.go index 89b8841217..c3512241f3 100644 --- a/go/vt/topo/test/keyspace.go +++ b/go/vt/topo/test/keyspace.go @@ -82,18 +82,18 @@ func CheckKeyspace(ctx context.Context, t *testing.T, ts topo.Impl) { if err := ts.DeleteKeyspaceShards(ctx, "test_keyspace2"); err != nil { t.Errorf("DeleteKeyspaceShards: %v", err) } - ki, err := ts.GetKeyspace(ctx, "test_keyspace2") + storedK, storedVersion, err := ts.GetKeyspace(ctx, "test_keyspace2") if err != nil { t.Fatalf("GetKeyspace: %v", err) } - if !reflect.DeepEqual(ki.Keyspace, k) { - t.Fatalf("returned keyspace doesn't match: got %v expected %v", ki.Keyspace, k) + if !reflect.DeepEqual(storedK, k) { + t.Fatalf("returned keyspace doesn't match: got %v expected %v", storedK, k) } - ki.ShardingColumnName = "other_id" - ki.ShardingColumnType = pb.KeyspaceIdType_BYTES + storedK.ShardingColumnName = "other_id" + storedK.ShardingColumnType = pb.KeyspaceIdType_BYTES var newServedFroms []*pb.Keyspace_ServedFrom - for _, ksf := range ki.ServedFroms { + for _, ksf := range storedK.ServedFroms { if ksf.TabletType == pb.TabletType_MASTER { continue } @@ -102,18 +102,20 @@ func CheckKeyspace(ctx context.Context, t *testing.T, ts topo.Impl) { } newServedFroms = append(newServedFroms, ksf) } - ki.ServedFroms = newServedFroms - _, err = ts.UpdateKeyspace(ctx, ki.KeyspaceName(), ki.Keyspace, ki.Version()) + storedK.ServedFroms = newServedFroms + _, err = ts.UpdateKeyspace(ctx, "test_keyspace2", storedK, storedVersion) if err != nil { t.Fatalf("UpdateKeyspace: %v", err) } - ki, err = ts.GetKeyspace(ctx, "test_keyspace2") + storedK, storedVersion, err = ts.GetKeyspace(ctx, "test_keyspace2") if err != nil { t.Fatalf("GetKeyspace: %v", err) } - if ki.ShardingColumnName != "other_id" || - ki.ShardingColumnType != pb.KeyspaceIdType_BYTES || - ki.GetServedFrom(pb.TabletType_REPLICA).Keyspace != "test_keyspace4" { - t.Errorf("GetKeyspace: unexpected keyspace, got %v", *ki) + if storedK.ShardingColumnName != "other_id" || + storedK.ShardingColumnType != pb.KeyspaceIdType_BYTES || + len(storedK.ServedFroms) != 1 || + storedK.ServedFroms[0].TabletType != pb.TabletType_REPLICA || + storedK.ServedFroms[0].Keyspace != "test_keyspace4" { + t.Errorf("GetKeyspace: unexpected keyspace, got %v", *storedK) } } diff --git a/go/vt/topotools/events/migrate.go b/go/vt/topotools/events/migrate.go index f6eda2d282..51cf32abc2 100644 --- a/go/vt/topotools/events/migrate.go +++ b/go/vt/topotools/events/migrate.go @@ -17,7 +17,7 @@ import ( type MigrateServedFrom struct { base.StatusUpdater - Keyspace topo.KeyspaceInfo + KeyspaceName string SourceShard topo.ShardInfo DestinationShard topo.ShardInfo ServedType pb.TabletType @@ -29,7 +29,7 @@ type MigrateServedFrom struct { type MigrateServedTypes struct { base.StatusUpdater - Keyspace topo.KeyspaceInfo + KeyspaceName string SourceShards []*topo.ShardInfo DestinationShards []*topo.ShardInfo ServedType pb.TabletType diff --git a/go/vt/topotools/events/migrate_syslog.go b/go/vt/topotools/events/migrate_syslog.go index f3e6193e70..3414d0c063 100644 --- a/go/vt/topotools/events/migrate_syslog.go +++ b/go/vt/topotools/events/migrate_syslog.go @@ -21,7 +21,7 @@ func (ev *MigrateServedFrom) Syslog() (syslog.Priority, string) { format = "%s [migrate served-from %s/%s -> %s/%s] %s" } return syslog.LOG_INFO, fmt.Sprintf(format, - ev.Keyspace.KeyspaceName(), + ev.KeyspaceName, ev.SourceShard.Keyspace(), ev.SourceShard.ShardName(), ev.DestinationShard.Keyspace(), ev.DestinationShard.ShardName(), ev.Status) @@ -48,7 +48,7 @@ func (ev *MigrateServedTypes) Syslog() (syslog.Priority, string) { } return syslog.LOG_INFO, fmt.Sprintf(format, - ev.Keyspace.KeyspaceName(), strings.Join(sourceShards, ", "), + ev.KeyspaceName, strings.Join(sourceShards, ", "), strings.Join(destShards, ", "), ev.Status) } diff --git a/go/vt/topotools/events/migrate_syslog_test.go b/go/vt/topotools/events/migrate_syslog_test.go index b1991aeb06..4326265b77 100644 --- a/go/vt/topotools/events/migrate_syslog_test.go +++ b/go/vt/topotools/events/migrate_syslog_test.go @@ -15,7 +15,7 @@ import ( func TestMigrateServedFromSyslogForward(t *testing.T) { wantSev, wantMsg := syslog.LOG_INFO, "keyspace-1 [migrate served-from keyspace-2/source-shard -> keyspace-1/dest-shard] status" ev := &MigrateServedFrom{ - Keyspace: *topo.NewKeyspaceInfo("keyspace-1", nil, -1), + KeyspaceName: "keyspace-1", SourceShard: *topo.NewShardInfo("keyspace-2", "source-shard", nil, -1), DestinationShard: *topo.NewShardInfo("keyspace-1", "dest-shard", nil, -1), Reverse: false, @@ -34,7 +34,7 @@ func TestMigrateServedFromSyslogForward(t *testing.T) { func TestMigrateServedFromSyslogReverse(t *testing.T) { wantSev, wantMsg := syslog.LOG_INFO, "keyspace-1 [migrate served-from keyspace-2/source-shard <- keyspace-1/dest-shard] status" ev := &MigrateServedFrom{ - Keyspace: *topo.NewKeyspaceInfo("keyspace-1", nil, -1), + KeyspaceName: "keyspace-1", SourceShard: *topo.NewShardInfo("keyspace-2", "source-shard", nil, -1), DestinationShard: *topo.NewShardInfo("keyspace-1", "dest-shard", nil, -1), Reverse: true, @@ -53,7 +53,7 @@ func TestMigrateServedFromSyslogReverse(t *testing.T) { func TestMigrateServedTypesSyslogForward(t *testing.T) { wantSev, wantMsg := syslog.LOG_INFO, "keyspace-1 [migrate served-types {src1, src2} -> {dst1, dst2}] status" ev := &MigrateServedTypes{ - Keyspace: *topo.NewKeyspaceInfo("keyspace-1", nil, -1), + KeyspaceName: "keyspace-1", SourceShards: []*topo.ShardInfo{ topo.NewShardInfo("keyspace-1", "src1", nil, -1), topo.NewShardInfo("keyspace-1", "src2", nil, -1), @@ -78,7 +78,7 @@ func TestMigrateServedTypesSyslogForward(t *testing.T) { func TestMigrateServedTypesSyslogReverse(t *testing.T) { wantSev, wantMsg := syslog.LOG_INFO, "keyspace-1 [migrate served-types {src1, src2} <- {dst1, dst2}] status" ev := &MigrateServedTypes{ - Keyspace: *topo.NewKeyspaceInfo("keyspace-1", nil, -1), + KeyspaceName: "keyspace-1", SourceShards: []*topo.ShardInfo{ topo.NewShardInfo("keyspace-1", "src1", nil, -1), topo.NewShardInfo("keyspace-1", "src2", nil, -1), diff --git a/go/vt/wrangler/keyspace.go b/go/vt/wrangler/keyspace.go index 9cdd5dc126..633dfb0864 100644 --- a/go/vt/wrangler/keyspace.go +++ b/go/vt/wrangler/keyspace.go @@ -310,7 +310,7 @@ func (wr *Wrangler) migrateServedTypes(ctx context.Context, keyspace string, sou } ev := &events.MigrateServedTypes{ - Keyspace: *topo.NewKeyspaceInfo(keyspace, nil, -1), + KeyspaceName: keyspace, SourceShards: sourceShards, DestinationShards: destinationShards, ServedType: servedType, @@ -550,7 +550,7 @@ func (wr *Wrangler) migrateServedFrom(ctx context.Context, ki *topo.KeyspaceInfo } ev := &events.MigrateServedFrom{ - Keyspace: *ki, + KeyspaceName: ki.KeyspaceName(), SourceShard: *sourceShard, DestinationShard: *destinationShard, ServedType: servedType, diff --git a/go/vt/zktopo/keyspace.go b/go/vt/zktopo/keyspace.go index 4956290a77..0e4aab5af2 100644 --- a/go/vt/zktopo/keyspace.go +++ b/go/vt/zktopo/keyspace.go @@ -10,9 +10,7 @@ import ( "path" "sort" - "github.com/youtube/vitess/go/event" "github.com/youtube/vitess/go/vt/topo" - "github.com/youtube/vitess/go/vt/topo/events" "github.com/youtube/vitess/go/zk" "golang.org/x/net/context" "launchpad.net/gozk/zookeeper" @@ -61,12 +59,6 @@ func (zkts *Server) CreateKeyspace(ctx context.Context, keyspace string, value * if alreadyExists { return topo.ErrNodeExists } - - event.Dispatch(&events.KeyspaceChange{ - KeyspaceName: keyspace, - Keyspace: value, - Status: "created", - }) return nil } @@ -98,32 +90,26 @@ func (zkts *Server) DeleteKeyspace(ctx context.Context, keyspace string) error { } return err } - - event.Dispatch(&events.KeyspaceChange{ - KeyspaceName: keyspace, - Keyspace: nil, - Status: "deleted", - }) return nil } // GetKeyspace is part of the topo.Server interface -func (zkts *Server) GetKeyspace(ctx context.Context, keyspace string) (*topo.KeyspaceInfo, error) { +func (zkts *Server) GetKeyspace(ctx context.Context, keyspace string) (*pb.Keyspace, int64, error) { keyspacePath := path.Join(globalKeyspacesPath, keyspace) data, stat, err := zkts.zconn.Get(keyspacePath) if err != nil { if zookeeper.IsError(err, zookeeper.ZNONODE) { err = topo.ErrNoNode } - return nil, err + return nil, 0, err } k := &pb.Keyspace{} if err = json.Unmarshal([]byte(data), k); err != nil { - return nil, fmt.Errorf("bad keyspace data %v", err) + return nil, 0, fmt.Errorf("bad keyspace data %v", err) } - return topo.NewKeyspaceInfo(keyspace, k, int64(stat.Version())), nil + return k, int64(stat.Version()), nil } // GetKeyspaces is part of the topo.Server interface From 5fd009a334d2372a31090b57e747ab23f89eaebb Mon Sep 17 00:00:00 2001 From: Alain Jobart Date: Thu, 20 Aug 2015 08:13:33 -0700 Subject: [PATCH 70/80] Converting UpdateShard to new style. --- go/vt/etcdtopo/shard.go | 13 ++------- go/vt/tabletmanager/healthcheck_test.go | 2 +- go/vt/tabletmanager/init_tablet.go | 2 +- go/vt/tabletmanager/init_tablet_test.go | 3 +- go/vt/topo/helpers/copy.go | 2 +- go/vt/topo/helpers/tee.go | 28 +++++++++--------- go/vt/topo/server.go | 2 +- go/vt/topo/shard.go | 29 +++++++++++++------ go/vt/topo/test/faketopo/faketopo.go | 2 +- go/vt/topo/test/shard.go | 2 +- go/vt/topotools/rebuild_test.go | 4 +-- go/vt/wrangler/keyspace.go | 12 ++++---- go/vt/wrangler/reparent.go | 6 ++-- go/vt/wrangler/shard.go | 12 ++++---- go/vt/wrangler/split.go | 2 +- go/vt/wrangler/tablet.go | 2 +- .../testlib/init_shard_master_test.go | 3 +- go/vt/wrangler/testlib/permissions_test.go | 3 +- .../testlib/reparent_external_test.go | 2 +- go/vt/wrangler/testlib/reparent_utils_test.go | 4 +-- go/vt/zktopo/shard.go | 13 ++------- 21 files changed, 71 insertions(+), 77 deletions(-) diff --git a/go/vt/etcdtopo/shard.go b/go/vt/etcdtopo/shard.go index 00033f5152..481560fe58 100644 --- a/go/vt/etcdtopo/shard.go +++ b/go/vt/etcdtopo/shard.go @@ -41,13 +41,13 @@ func (s *Server) CreateShard(ctx context.Context, keyspace, shard string, value } // UpdateShard implements topo.Server. -func (s *Server) UpdateShard(ctx context.Context, si *topo.ShardInfo, existingVersion int64) (int64, error) { - data, err := json.MarshalIndent(si.Shard, "", " ") +func (s *Server) UpdateShard(ctx context.Context, keyspace, shard string, value *pb.Shard, existingVersion int64) (int64, error) { + data, err := json.MarshalIndent(value, "", " ") if err != nil { return -1, err } - resp, err := s.getGlobal().CompareAndSwap(shardFilePath(si.Keyspace(), si.ShardName()), + resp, err := s.getGlobal().CompareAndSwap(shardFilePath(keyspace, shard), string(data), 0 /* ttl */, "" /* prevValue */, uint64(existingVersion)) if err != nil { return -1, convertError(err) @@ -55,13 +55,6 @@ func (s *Server) UpdateShard(ctx context.Context, si *topo.ShardInfo, existingVe if resp.Node == nil { return -1, ErrBadResponse } - - event.Dispatch(&events.ShardChange{ - KeyspaceName: si.Keyspace(), - ShardName: si.ShardName(), - Shard: si.Shard, - Status: "updated", - }) return int64(resp.Node.ModifiedIndex), nil } diff --git a/go/vt/tabletmanager/healthcheck_test.go b/go/vt/tabletmanager/healthcheck_test.go index 9bf64236c8..3f2548eaa5 100644 --- a/go/vt/tabletmanager/healthcheck_test.go +++ b/go/vt/tabletmanager/healthcheck_test.go @@ -301,7 +301,7 @@ func TestTabletControl(t *testing.T) { DisableQueryService: true, }, } - if err := topo.UpdateShard(ctx, agent.TopoServer, si); err != nil { + if err := agent.TopoServer.UpdateShard(ctx, si); err != nil { t.Fatalf("UpdateShard failed: %v", err) } diff --git a/go/vt/tabletmanager/init_tablet.go b/go/vt/tabletmanager/init_tablet.go index 24ee26f3f5..aa66a3ec92 100644 --- a/go/vt/tabletmanager/init_tablet.go +++ b/go/vt/tabletmanager/init_tablet.go @@ -130,7 +130,7 @@ func (agent *ActionAgent) InitTablet(port, gRPCPort int32) error { si.Cells = append(si.Cells, agent.TabletAlias.Cell) // write it back - if err := topo.UpdateShard(ctx, agent.TopoServer, si); err != nil { + if err := agent.TopoServer.UpdateShard(ctx, si); err != nil { return actionNode.UnlockShard(ctx, agent.TopoServer, *initKeyspace, shard, lockPath, err) } } diff --git a/go/vt/tabletmanager/init_tablet_test.go b/go/vt/tabletmanager/init_tablet_test.go index 7b9febaa8b..e2513a2e74 100644 --- a/go/vt/tabletmanager/init_tablet_test.go +++ b/go/vt/tabletmanager/init_tablet_test.go @@ -13,7 +13,6 @@ import ( "github.com/youtube/vitess/go/history" "github.com/youtube/vitess/go/stats" "github.com/youtube/vitess/go/vt/mysqlctl" - "github.com/youtube/vitess/go/vt/topo" "github.com/youtube/vitess/go/vt/zktopo" "golang.org/x/net/context" @@ -140,7 +139,7 @@ func TestInitTablet(t *testing.T) { t.Fatalf("GetShard failed: %v", err) } si.MasterAlias = tabletAlias - if err := topo.UpdateShard(ctx, ts, si); err != nil { + if err := ts.UpdateShard(ctx, si); err != nil { t.Fatalf("UpdateShard failed: %v", err) } if err := agent.InitTablet(port, gRPCPort); err != nil { diff --git a/go/vt/topo/helpers/copy.go b/go/vt/topo/helpers/copy.go index bb6945fc06..033b380a44 100644 --- a/go/vt/topo/helpers/copy.go +++ b/go/vt/topo/helpers/copy.go @@ -104,7 +104,7 @@ func CopyShards(ctx context.Context, fromTS, toTS topo.Impl, deleteKeyspaceShard return } - if _, err := toTS.UpdateShard(ctx, si, toSi.Version()); err != nil { + if _, err := toTS.UpdateShard(ctx, keyspace, shard, si.Shard, toSi.Version()); err != nil { rec.RecordError(fmt.Errorf("UpdateShard(%v, %v): %v", keyspace, shard, err)) } }(keyspace, shard) diff --git a/go/vt/topo/helpers/tee.go b/go/vt/topo/helpers/tee.go index c0bcf16056..a229123a91 100644 --- a/go/vt/topo/helpers/tee.go +++ b/go/vt/topo/helpers/tee.go @@ -241,8 +241,8 @@ func (tee *Tee) CreateShard(ctx context.Context, keyspace, shard string, value * } // UpdateShard is part of the topo.Server interface -func (tee *Tee) UpdateShard(ctx context.Context, si *topo.ShardInfo, existingVersion int64) (newVersion int64, err error) { - if newVersion, err = tee.primary.UpdateShard(ctx, si, existingVersion); err != nil { +func (tee *Tee) UpdateShard(ctx context.Context, keyspace, shard string, value *pb.Shard, existingVersion int64) (newVersion int64, err error) { + if newVersion, err = tee.primary.UpdateShard(ctx, keyspace, shard, value, existingVersion); err != nil { // failed on primary, not updating secondary return } @@ -251,39 +251,39 @@ func (tee *Tee) UpdateShard(ctx context.Context, si *topo.ShardInfo, existingVer // and shard version in second topo, replace the version number. // if not, this will probably fail and log. tee.mu.Lock() - svm, ok := tee.shardVersionMapping[si.Keyspace()+"/"+si.ShardName()] + svm, ok := tee.shardVersionMapping[keyspace+"/"+shard] if ok && svm.readFromVersion == existingVersion { existingVersion = svm.readFromSecondVersion - delete(tee.shardVersionMapping, si.Keyspace()+"/"+si.ShardName()) + delete(tee.shardVersionMapping, keyspace+"/"+shard) } tee.mu.Unlock() - if newVersion2, serr := tee.secondary.UpdateShard(ctx, si, existingVersion); serr != nil { + if newVersion2, serr := tee.secondary.UpdateShard(ctx, keyspace, shard, value, existingVersion); serr != nil { // not critical enough to fail if serr == topo.ErrNoNode { // the shard doesn't exist on the secondary, let's // just create it - if serr = tee.secondary.CreateShard(ctx, si.Keyspace(), si.ShardName(), si.Shard); serr != nil { - log.Warningf("secondary.CreateShard(%v,%v) failed (after UpdateShard returned ErrNoNode): %v", si.Keyspace(), si.ShardName(), serr) + if serr = tee.secondary.CreateShard(ctx, keyspace, shard, value); serr != nil { + log.Warningf("secondary.CreateShard(%v,%v) failed (after UpdateShard returned ErrNoNode): %v", keyspace, shard, serr) } else { - log.Infof("secondary.UpdateShard(%v, %v) failed with ErrNoNode, CreateShard then worked.", si.Keyspace(), si.ShardName()) - si, gerr := tee.secondary.GetShard(ctx, si.Keyspace(), si.ShardName()) + log.Infof("secondary.UpdateShard(%v, %v) failed with ErrNoNode, CreateShard then worked.", keyspace, shard) + s, gerr := tee.secondary.GetShard(ctx, keyspace, shard) if gerr != nil { - log.Warningf("Failed to re-read shard(%v, %v) after creating it on secondary: %v", si.Keyspace(), si.ShardName(), gerr) + log.Warningf("Failed to re-read shard(%v, %v) after creating it on secondary: %v", keyspace, shard, gerr) } else { tee.mu.Lock() - tee.shardVersionMapping[si.Keyspace()+"/"+si.ShardName()] = versionMapping{ + tee.shardVersionMapping[keyspace+"/"+shard] = versionMapping{ readFromVersion: newVersion, - readFromSecondVersion: si.Version(), + readFromSecondVersion: s.Version(), } tee.mu.Unlock() } } } else { - log.Warningf("secondary.UpdateShard(%v, %v) failed: %v", si.Keyspace(), si.ShardName(), serr) + log.Warningf("secondary.UpdateShard(%v, %v) failed: %v", keyspace, shard, serr) } } else { tee.mu.Lock() - tee.shardVersionMapping[si.Keyspace()+"/"+si.ShardName()] = versionMapping{ + tee.shardVersionMapping[keyspace+"/"+shard] = versionMapping{ readFromVersion: newVersion, readFromSecondVersion: newVersion2, } diff --git a/go/vt/topo/server.go b/go/vt/topo/server.go index c234a80fb6..5ad31b05c4 100644 --- a/go/vt/topo/server.go +++ b/go/vt/topo/server.go @@ -115,7 +115,7 @@ type Impl interface { // or ErrBadVersion if the version has changed. // // Do not use directly, but instead use topo.UpdateShard. - UpdateShard(ctx context.Context, si *ShardInfo, existingVersion int64) (newVersion int64, err error) + UpdateShard(ctx context.Context, keyspace, shard string, value *pb.Shard, existingVersion int64) (newVersion int64, err error) // ValidateShard performs routine checks on the shard. ValidateShard(ctx context.Context, keyspace, shard string) error diff --git a/go/vt/topo/shard.go b/go/vt/topo/shard.go index c0ea53747d..c5ed70317e 100644 --- a/go/vt/topo/shard.go +++ b/go/vt/topo/shard.go @@ -17,9 +17,11 @@ import ( log "github.com/golang/glog" + "github.com/youtube/vitess/go/event" "github.com/youtube/vitess/go/trace" "github.com/youtube/vitess/go/vt/concurrency" "github.com/youtube/vitess/go/vt/key" + "github.com/youtube/vitess/go/vt/topo/events" "github.com/youtube/vitess/go/vt/topo/topoproto" pb "github.com/youtube/vitess/go/vt/proto/topodata" @@ -201,12 +203,13 @@ func (ts Server) GetShard(ctx context.Context, keyspace, shard string) (*ShardIn return ts.Impl.GetShard(ctx, keyspace, shard) } -// UpdateShard updates the shard data, with the right version -func UpdateShard(ctx context.Context, ts Server, si *ShardInfo) error { +// UpdateShard updates the shard data, with the right version. +// It also creates a span, and dispatches the event. +func (ts Server) UpdateShard(ctx context.Context, si *ShardInfo) error { span := trace.NewSpanFromContext(ctx) span.StartClient("TopoServer.UpdateShard") - span.Annotate("keyspace", si.Keyspace()) - span.Annotate("shard", si.ShardName()) + span.Annotate("keyspace", si.keyspace) + span.Annotate("shard", si.shardName) defer span.Finish() var version int64 = -1 @@ -214,11 +217,19 @@ func UpdateShard(ctx context.Context, ts Server, si *ShardInfo) error { version = si.version } - newVersion, err := ts.UpdateShard(ctx, si, version) - if err == nil { - si.version = newVersion + newVersion, err := ts.Impl.UpdateShard(ctx, si.keyspace, si.shardName, si.Shard, version) + if err != nil { + return err } - return err + si.version = newVersion + + event.Dispatch(&events.ShardChange{ + KeyspaceName: si.Keyspace(), + ShardName: si.ShardName(), + Shard: si.Shard, + Status: "updated", + }) + return nil } // UpdateShardFields is a high level helper to read a shard record, call an @@ -234,7 +245,7 @@ func UpdateShardFields(ctx context.Context, ts Server, keyspace, shard string, u if err = update(si.Shard); err != nil { return nil, err } - if err = UpdateShard(ctx, ts, si); err != ErrBadVersion { + if err = ts.UpdateShard(ctx, si); err != ErrBadVersion { return si, err } } diff --git a/go/vt/topo/test/faketopo/faketopo.go b/go/vt/topo/test/faketopo/faketopo.go index b83fdb022d..2ca6b10fbc 100644 --- a/go/vt/topo/test/faketopo/faketopo.go +++ b/go/vt/topo/test/faketopo/faketopo.go @@ -75,7 +75,7 @@ func (ft FakeTopo) CreateShard(ctx context.Context, keyspace, shard string, valu } // UpdateShard implements topo.Server. -func (ft FakeTopo) UpdateShard(ctx context.Context, si *topo.ShardInfo, existingVersion int64) (int64, error) { +func (ft FakeTopo) UpdateShard(ctx context.Context, keyspace, shard string, value *pb.Shard, existingVersion int64) (int64, error) { return 0, errNotImplemented } diff --git a/go/vt/topo/test/shard.go b/go/vt/topo/test/shard.go index 53ed96fdc5..8b6d8fbe87 100644 --- a/go/vt/topo/test/shard.go +++ b/go/vt/topo/test/shard.go @@ -105,7 +105,7 @@ func CheckShard(ctx context.Context, t *testing.T, ts topo.Impl) { DisableQueryService: true, }, } - if err := topo.UpdateShard(ctx, tts, shardInfo); err != nil { + if err := tts.UpdateShard(ctx, shardInfo); err != nil { t.Errorf("UpdateShard: %v", err) } diff --git a/go/vt/topotools/rebuild_test.go b/go/vt/topotools/rebuild_test.go index 3b28c1c419..574dc0be1a 100644 --- a/go/vt/topotools/rebuild_test.go +++ b/go/vt/topotools/rebuild_test.go @@ -62,7 +62,7 @@ func TestRebuildShard(t *testing.T) { t.Fatalf("GetOrCreateShard: %v", err) } si.Cells = append(si.Cells, cells[0]) - if err := topo.UpdateShard(ctx, ts, si); err != nil { + if err := ts.UpdateShard(ctx, si); err != nil { t.Fatalf("UpdateShard: %v", err) } @@ -128,7 +128,7 @@ func TestUpdateTabletEndpoints(t *testing.T) { t.Fatalf("GetOrCreateShard: %v", err) } si.Cells = append(si.Cells, cell) - if err := topo.UpdateShard(ctx, ts, si); err != nil { + if err := ts.UpdateShard(ctx, si); err != nil { t.Fatalf("UpdateShard: %v", err) } diff --git a/go/vt/wrangler/keyspace.go b/go/vt/wrangler/keyspace.go index 633dfb0864..38a46e524b 100644 --- a/go/vt/wrangler/keyspace.go +++ b/go/vt/wrangler/keyspace.go @@ -334,7 +334,7 @@ func (wr *Wrangler) migrateServedTypes(ctx context.Context, keyspace string, sou if err := si.UpdateDisableQueryService(pb.TabletType_MASTER, nil, true); err != nil { return err } - if err := topo.UpdateShard(ctx, wr.ts, si); err != nil { + if err := wr.ts.UpdateShard(ctx, si); err != nil { return err } } @@ -413,7 +413,7 @@ func (wr *Wrangler) migrateServedTypes(ctx context.Context, keyspace string, sou // All is good, we can save the shards now event.DispatchUpdate(ev, "updating source shards") for _, si := range sourceShards { - if err := topo.UpdateShard(ctx, wr.ts, si); err != nil { + if err := wr.ts.UpdateShard(ctx, si); err != nil { return err } } @@ -425,7 +425,7 @@ func (wr *Wrangler) migrateServedTypes(ctx context.Context, keyspace string, sou } event.DispatchUpdate(ev, "updating destination shards") for _, si := range destinationShards { - if err := topo.UpdateShard(ctx, wr.ts, si); err != nil { + if err := wr.ts.UpdateShard(ctx, si); err != nil { return err } } @@ -585,7 +585,7 @@ func (wr *Wrangler) replicaMigrateServedFrom(ctx context.Context, ki *topo.Keysp if err := sourceShard.UpdateSourceBlacklistedTables(servedType, cells, reverse, tables); err != nil { return fmt.Errorf("UpdateSourceBlacklistedTables(%v/%v) failed: %v", sourceShard.Keyspace(), sourceShard.ShardName(), err) } - if err := topo.UpdateShard(ctx, wr.ts, sourceShard); err != nil { + if err := wr.ts.UpdateShard(ctx, sourceShard); err != nil { return fmt.Errorf("UpdateShard(%v/%v) failed: %v", sourceShard.Keyspace(), sourceShard.ShardName(), err) } @@ -625,7 +625,7 @@ func (wr *Wrangler) masterMigrateServedFrom(ctx context.Context, ki *topo.Keyspa if err := sourceShard.UpdateSourceBlacklistedTables(pb.TabletType_MASTER, nil, false, tables); err != nil { return fmt.Errorf("UpdateSourceBlacklistedTables(%v/%v) failed: %v", sourceShard.Keyspace(), sourceShard.ShardName(), err) } - if err := topo.UpdateShard(ctx, wr.ts, sourceShard); err != nil { + if err := wr.ts.UpdateShard(ctx, sourceShard); err != nil { return fmt.Errorf("UpdateShard(%v/%v) failed: %v", sourceShard.Keyspace(), sourceShard.ShardName(), err) } @@ -660,7 +660,7 @@ func (wr *Wrangler) masterMigrateServedFrom(ctx context.Context, ki *topo.Keyspa // Update the destination shard (no more source shard) event.DispatchUpdate(ev, "updating destination shard") destinationShard.SourceShards = nil - if err := topo.UpdateShard(ctx, wr.ts, destinationShard); err != nil { + if err := wr.ts.UpdateShard(ctx, destinationShard); err != nil { return err } diff --git a/go/vt/wrangler/reparent.go b/go/vt/wrangler/reparent.go index bb9779a8e5..a63f3aa77a 100644 --- a/go/vt/wrangler/reparent.go +++ b/go/vt/wrangler/reparent.go @@ -290,7 +290,7 @@ func (wr *Wrangler) initShardMasterLocked(ctx context.Context, ev *events.Repare } if !topoproto.TabletAliasEqual(shardInfo.MasterAlias, masterElectTabletAlias) { shardInfo.MasterAlias = masterElectTabletAlias - if err := topo.UpdateShard(ctx, wr.ts, shardInfo); err != nil { + if err := wr.ts.UpdateShard(ctx, shardInfo); err != nil { wgSlaves.Wait() return fmt.Errorf("failed to update shard master record: %v", err) } @@ -423,7 +423,7 @@ func (wr *Wrangler) plannedReparentShardLocked(ctx context.Context, ev *events.R } wr.logger.Infof("updating shard record with new master %v", masterElectTabletAlias) shardInfo.MasterAlias = masterElectTabletAlias - if err := topo.UpdateShard(ctx, wr.ts, shardInfo); err != nil { + if err := wr.ts.UpdateShard(ctx, shardInfo); err != nil { wgSlaves.Wait() return fmt.Errorf("failed to update shard master record: %v", err) } @@ -615,7 +615,7 @@ func (wr *Wrangler) emergencyReparentShardLocked(ctx context.Context, ev *events } wr.logger.Infof("updating shard record with new master %v", topoproto.TabletAliasString(masterElectTabletAlias)) shardInfo.MasterAlias = masterElectTabletAlias - if err := topo.UpdateShard(ctx, wr.ts, shardInfo); err != nil { + if err := wr.ts.UpdateShard(ctx, shardInfo); err != nil { wgSlaves.Wait() return fmt.Errorf("failed to update shard master record: %v", err) } diff --git a/go/vt/wrangler/shard.go b/go/vt/wrangler/shard.go index 60ce01fbc3..e323dc62bf 100644 --- a/go/vt/wrangler/shard.go +++ b/go/vt/wrangler/shard.go @@ -74,7 +74,7 @@ func (wr *Wrangler) updateShardCellsAndMaster(ctx context.Context, si *topo.Shar if wasUpdated { // write it back - if err := topo.UpdateShard(ctx, wr.ts, si); err != nil { + if err := wr.ts.UpdateShard(ctx, si); err != nil { return wr.unlockShard(ctx, keyspace, shard, actionNode, lockPath, err) } } @@ -106,7 +106,7 @@ func (wr *Wrangler) setShardServedTypes(ctx context.Context, keyspace, shard str if err := si.UpdateServedTypesMap(servedType, cells, remove); err != nil { return err } - return topo.UpdateShard(ctx, wr.ts, si) + return wr.ts.UpdateShard(ctx, si) } // SetShardTabletControl changes the TabletControl records @@ -148,7 +148,7 @@ func (wr *Wrangler) setShardTabletControl(ctx context.Context, keyspace, shard s return fmt.Errorf("UpdateSourceBlacklistedTables(%v/%v) failed: %v", shardInfo.Keyspace(), shardInfo.ShardName(), err) } } - return topo.UpdateShard(ctx, wr.ts, shardInfo) + return wr.ts.UpdateShard(ctx, shardInfo) } // DeleteShard will do all the necessary changes in the topology server @@ -296,7 +296,7 @@ func (wr *Wrangler) removeShardCell(ctx context.Context, keyspace, shard, cell s } shardInfo.Cells = newCells - return topo.UpdateShard(ctx, wr.ts, shardInfo) + return wr.ts.UpdateShard(ctx, shardInfo) } // SourceShardDelete will delete a SourceShard inside a shard, by index. @@ -329,7 +329,7 @@ func (wr *Wrangler) sourceShardDelete(ctx context.Context, keyspace, shard strin newSourceShards = nil } si.SourceShards = newSourceShards - return topo.UpdateShard(ctx, wr.ts, si) + return wr.ts.UpdateShard(ctx, si) } // SourceShardAdd will add a new SourceShard inside a shard @@ -364,5 +364,5 @@ func (wr *Wrangler) sourceShardAdd(ctx context.Context, keyspace, shard string, KeyRange: keyRange, Tables: tables, }) - return topo.UpdateShard(ctx, wr.ts, si) + return wr.ts.UpdateShard(ctx, si) } diff --git a/go/vt/wrangler/split.go b/go/vt/wrangler/split.go index fd502083dc..b6c0712b0c 100644 --- a/go/vt/wrangler/split.go +++ b/go/vt/wrangler/split.go @@ -51,7 +51,7 @@ func (wr *Wrangler) SetSourceShards(ctx context.Context, keyspace, shard string, } // and write the shard - if err = topo.UpdateShard(ctx, wr.ts, shardInfo); err != nil { + if err = wr.ts.UpdateShard(ctx, shardInfo); err != nil { return err } diff --git a/go/vt/wrangler/tablet.go b/go/vt/wrangler/tablet.go index 3604aab2f1..6de86ce059 100644 --- a/go/vt/wrangler/tablet.go +++ b/go/vt/wrangler/tablet.go @@ -158,7 +158,7 @@ func (wr *Wrangler) Scrap(ctx context.Context, tabletAlias *pb.TabletAlias, forc si.MasterAlias = nil // write it back - if err := topo.UpdateShard(ctx, wr.ts, si); err != nil { + if err := wr.ts.UpdateShard(ctx, si); err != nil { return wr.unlockShard(ctx, ti.Keyspace, ti.Shard, actionNode, lockPath, err) } } else { diff --git a/go/vt/wrangler/testlib/init_shard_master_test.go b/go/vt/wrangler/testlib/init_shard_master_test.go index b9cb7a2312..55fae61aaf 100644 --- a/go/vt/wrangler/testlib/init_shard_master_test.go +++ b/go/vt/wrangler/testlib/init_shard_master_test.go @@ -12,7 +12,6 @@ import ( "github.com/youtube/vitess/go/vt/logutil" myproto "github.com/youtube/vitess/go/vt/mysqlctl/proto" "github.com/youtube/vitess/go/vt/tabletmanager/tmclient" - "github.com/youtube/vitess/go/vt/topo" "github.com/youtube/vitess/go/vt/topo/topoproto" "github.com/youtube/vitess/go/vt/wrangler" "github.com/youtube/vitess/go/vt/zktopo" @@ -220,7 +219,7 @@ func TestInitMasterShardOneSlaveFails(t *testing.T) { t.Fatalf("GetShard failed: %v", err) } si.MasterAlias.Uid++ - if err := topo.UpdateShard(ctx, ts, si); err != nil { + if err := ts.UpdateShard(ctx, si); err != nil { t.Fatalf("UpdateShard failed: %v", err) } diff --git a/go/vt/wrangler/testlib/permissions_test.go b/go/vt/wrangler/testlib/permissions_test.go index 14b9714ea2..cbef5eae82 100644 --- a/go/vt/wrangler/testlib/permissions_test.go +++ b/go/vt/wrangler/testlib/permissions_test.go @@ -13,7 +13,6 @@ import ( "github.com/youtube/vitess/go/sqltypes" "github.com/youtube/vitess/go/vt/logutil" "github.com/youtube/vitess/go/vt/tabletmanager/tmclient" - "github.com/youtube/vitess/go/vt/topo" "github.com/youtube/vitess/go/vt/wrangler" "github.com/youtube/vitess/go/vt/zktopo" "golang.org/x/net/context" @@ -38,7 +37,7 @@ func TestPermissions(t *testing.T) { t.Fatalf("GetShard failed: %v", err) } si.MasterAlias = master.Tablet.Alias - if err := topo.UpdateShard(ctx, ts, si); err != nil { + if err := ts.UpdateShard(ctx, si); err != nil { t.Fatalf("UpdateShard failed: %v", err) } diff --git a/go/vt/wrangler/testlib/reparent_external_test.go b/go/vt/wrangler/testlib/reparent_external_test.go index a2b5734c8e..4a07da5d4c 100644 --- a/go/vt/wrangler/testlib/reparent_external_test.go +++ b/go/vt/wrangler/testlib/reparent_external_test.go @@ -49,7 +49,7 @@ func TestTabletExternallyReparented(t *testing.T) { t.Fatalf("GetShard failed: %v", err) } si.Cells = append(si.Cells, "cell666") - if err := topo.UpdateShard(ctx, ts, si); err != nil { + if err := ts.UpdateShard(ctx, si); err != nil { t.Fatalf("UpdateShard failed: %v", err) } diff --git a/go/vt/wrangler/testlib/reparent_utils_test.go b/go/vt/wrangler/testlib/reparent_utils_test.go index 0e9d480fa9..cf8f9cf706 100644 --- a/go/vt/wrangler/testlib/reparent_utils_test.go +++ b/go/vt/wrangler/testlib/reparent_utils_test.go @@ -39,7 +39,7 @@ func TestShardReplicationStatuses(t *testing.T) { t.Fatalf("GetShard failed: %v", err) } si.MasterAlias = master.Tablet.Alias - if err := topo.UpdateShard(ctx, ts, si); err != nil { + if err := ts.UpdateShard(ctx, si); err != nil { t.Fatalf("UpdateShard failed: %v", err) } @@ -107,7 +107,7 @@ func TestReparentTablet(t *testing.T) { t.Fatalf("GetShard failed: %v", err) } si.MasterAlias = master.Tablet.Alias - if err := topo.UpdateShard(ctx, ts, si); err != nil { + if err := ts.UpdateShard(ctx, si); err != nil { t.Fatalf("UpdateShard failed: %v", err) } diff --git a/go/vt/zktopo/shard.go b/go/vt/zktopo/shard.go index 3337466e37..e9acddb0f4 100644 --- a/go/vt/zktopo/shard.go +++ b/go/vt/zktopo/shard.go @@ -67,9 +67,9 @@ func (zkts *Server) CreateShard(ctx context.Context, keyspace, shard string, val } // UpdateShard is part of the topo.Server interface -func (zkts *Server) UpdateShard(ctx context.Context, si *topo.ShardInfo, existingVersion int64) (int64, error) { - shardPath := path.Join(globalKeyspacesPath, si.Keyspace(), "shards", si.ShardName()) - data, err := json.MarshalIndent(si.Shard, "", " ") +func (zkts *Server) UpdateShard(ctx context.Context, keyspace, shard string, value *pb.Shard, existingVersion int64) (int64, error) { + shardPath := path.Join(globalKeyspacesPath, keyspace, "shards", shard) + data, err := json.MarshalIndent(value, "", " ") if err != nil { return -1, err } @@ -80,13 +80,6 @@ func (zkts *Server) UpdateShard(ctx context.Context, si *topo.ShardInfo, existin } return -1, err } - - event.Dispatch(&events.ShardChange{ - KeyspaceName: si.Keyspace(), - ShardName: si.ShardName(), - Shard: si.Shard, - Status: "updated", - }) return int64(stat.Version()), nil } From 4966f2fd3c7bbddd3537ed7b790ccaba1e38214f Mon Sep 17 00:00:00 2001 From: Alain Jobart Date: Thu, 20 Aug 2015 08:34:21 -0700 Subject: [PATCH 71/80] Now fixing GetShard. --- go/vt/etcdtopo/shard.go | 13 +++++---- go/vt/schemamanager/schemamanager_test.go | 4 +-- go/vt/topo/helpers/copy.go | 10 +++---- go/vt/topo/helpers/copy_test.go | 6 ++--- go/vt/topo/helpers/tee.go | 20 +++++++------- go/vt/topo/server.go | 4 +-- go/vt/topo/shard.go | 11 +++++++- go/vt/topo/test/faketopo/faketopo.go | 4 +-- go/vt/topo/test/shard.go | 32 +++++++++++------------ go/vt/zktopo/shard.go | 8 +++--- 10 files changed, 60 insertions(+), 52 deletions(-) diff --git a/go/vt/etcdtopo/shard.go b/go/vt/etcdtopo/shard.go index 481560fe58..cb09538dcc 100644 --- a/go/vt/etcdtopo/shard.go +++ b/go/vt/etcdtopo/shard.go @@ -9,7 +9,6 @@ import ( "fmt" "github.com/youtube/vitess/go/event" - "github.com/youtube/vitess/go/vt/topo" "github.com/youtube/vitess/go/vt/topo/events" "golang.org/x/net/context" @@ -60,26 +59,26 @@ func (s *Server) UpdateShard(ctx context.Context, keyspace, shard string, value // ValidateShard implements topo.Server. func (s *Server) ValidateShard(ctx context.Context, keyspace, shard string) error { - _, err := s.GetShard(ctx, keyspace, shard) + _, _, err := s.GetShard(ctx, keyspace, shard) return err } // GetShard implements topo.Server. -func (s *Server) GetShard(ctx context.Context, keyspace, shard string) (*topo.ShardInfo, error) { +func (s *Server) GetShard(ctx context.Context, keyspace, shard string) (*pb.Shard, int64, error) { resp, err := s.getGlobal().Get(shardFilePath(keyspace, shard), false /* sort */, false /* recursive */) if err != nil { - return nil, convertError(err) + return nil, 0, convertError(err) } if resp.Node == nil { - return nil, ErrBadResponse + return nil, 0, ErrBadResponse } value := &pb.Shard{} if err := json.Unmarshal([]byte(resp.Node.Value), value); err != nil { - return nil, fmt.Errorf("bad shard data (%v): %q", err, resp.Node.Value) + return nil, 0, fmt.Errorf("bad shard data (%v): %q", err, resp.Node.Value) } - return topo.NewShardInfo(keyspace, shard, value, int64(resp.Node.ModifiedIndex)), nil + return value, int64(resp.Node.ModifiedIndex), nil } // GetShardNames implements topo.Server. diff --git a/go/vt/schemamanager/schemamanager_test.go b/go/vt/schemamanager/schemamanager_test.go index 3388328763..183b1ded1f 100644 --- a/go/vt/schemamanager/schemamanager_test.go +++ b/go/vt/schemamanager/schemamanager_test.go @@ -286,7 +286,7 @@ func (topoServer *fakeTopo) GetShardNames(ctx context.Context, keyspace string) return []string{"0", "1", "2"}, nil } -func (topoServer *fakeTopo) GetShard(ctx context.Context, keyspace string, shard string) (*topo.ShardInfo, error) { +func (topoServer *fakeTopo) GetShard(ctx context.Context, keyspace string, shard string) (*pb.Shard, int64, error) { var masterAlias *pb.TabletAlias if !topoServer.WithEmptyMasterAlias { masterAlias = &pb.TabletAlias{ @@ -297,7 +297,7 @@ func (topoServer *fakeTopo) GetShard(ctx context.Context, keyspace string, shard value := &pb.Shard{ MasterAlias: masterAlias, } - return topo.NewShardInfo(keyspace, shard, value, 0), nil + return value, 0, nil } func (topoServer *fakeTopo) GetTablet(ctx context.Context, tabletAlias *pb.TabletAlias) (*topo.TabletInfo, error) { diff --git a/go/vt/topo/helpers/copy.go b/go/vt/topo/helpers/copy.go index 033b380a44..05ca21a00a 100644 --- a/go/vt/topo/helpers/copy.go +++ b/go/vt/topo/helpers/copy.go @@ -92,19 +92,19 @@ func CopyShards(ctx context.Context, fromTS, toTS topo.Impl, deleteKeyspaceShard } } - si, err := fromTS.GetShard(ctx, keyspace, shard) + s, _, err := fromTS.GetShard(ctx, keyspace, shard) if err != nil { rec.RecordError(fmt.Errorf("GetShard(%v, %v): %v", keyspace, shard, err)) return } - toSi, err := toTS.GetShard(ctx, keyspace, shard) + _, toV, err := toTS.GetShard(ctx, keyspace, shard) if err != nil { rec.RecordError(fmt.Errorf("toTS.GetShard(%v, %v): %v", keyspace, shard, err)) return } - if _, err := toTS.UpdateShard(ctx, keyspace, shard, si.Shard, toSi.Version()); err != nil { + if _, err := toTS.UpdateShard(ctx, keyspace, shard, s, toV); err != nil { rec.RecordError(fmt.Errorf("UpdateShard(%v, %v): %v", keyspace, shard, err)) } }(keyspace, shard) @@ -197,13 +197,13 @@ func CopyShardReplications(ctx context.Context, fromTS, toTS topo.Impl) { defer wg.Done() // read the source shard to get the cells - si, err := fromTS.GetShard(ctx, keyspace, shard) + s, _, err := fromTS.GetShard(ctx, keyspace, shard) if err != nil { rec.RecordError(fmt.Errorf("GetShard(%v, %v): %v", keyspace, shard, err)) return } - for _, cell := range si.Cells { + for _, cell := range s.Cells { sri, err := fromTS.GetShardReplication(ctx, cell, keyspace, shard) if err != nil { rec.RecordError(fmt.Errorf("GetShardReplication(%v, %v, %v): %v", cell, keyspace, shard, err)) diff --git a/go/vt/topo/helpers/copy_test.go b/go/vt/topo/helpers/copy_test.go index 7196687f59..307da3fe59 100644 --- a/go/vt/topo/helpers/copy_test.go +++ b/go/vt/topo/helpers/copy_test.go @@ -117,12 +117,12 @@ func TestBasic(t *testing.T) { t.Fatalf("unexpected shards: %v", shards) } CopyShards(ctx, fromTS, toTS, false) - si, err := toTS.GetShard(ctx, "test_keyspace", "0") + s, _, err := toTS.GetShard(ctx, "test_keyspace", "0") if err != nil { t.Fatalf("cannot read shard: %v", err) } - if len(si.Cells) != 1 || si.Cells[0] != "test_cell" { - t.Fatalf("bad shard data: %v", *si) + if len(s.Cells) != 1 || s.Cells[0] != "test_cell" { + t.Fatalf("bad shard data: %v", *s) } // check ShardReplication copy diff --git a/go/vt/topo/helpers/tee.go b/go/vt/topo/helpers/tee.go index a229123a91..3c9f5484cf 100644 --- a/go/vt/topo/helpers/tee.go +++ b/go/vt/topo/helpers/tee.go @@ -266,14 +266,14 @@ func (tee *Tee) UpdateShard(ctx context.Context, keyspace, shard string, value * log.Warningf("secondary.CreateShard(%v,%v) failed (after UpdateShard returned ErrNoNode): %v", keyspace, shard, serr) } else { log.Infof("secondary.UpdateShard(%v, %v) failed with ErrNoNode, CreateShard then worked.", keyspace, shard) - s, gerr := tee.secondary.GetShard(ctx, keyspace, shard) + _, v, gerr := tee.secondary.GetShard(ctx, keyspace, shard) if gerr != nil { log.Warningf("Failed to re-read shard(%v, %v) after creating it on secondary: %v", keyspace, shard, gerr) } else { tee.mu.Lock() tee.shardVersionMapping[keyspace+"/"+shard] = versionMapping{ readFromVersion: newVersion, - readFromSecondVersion: s.Version(), + readFromSecondVersion: v, } tee.mu.Unlock() } @@ -307,25 +307,25 @@ func (tee *Tee) ValidateShard(ctx context.Context, keyspace, shard string) error } // GetShard is part of the topo.Server interface -func (tee *Tee) GetShard(ctx context.Context, keyspace, shard string) (*topo.ShardInfo, error) { - si, err := tee.readFrom.GetShard(ctx, keyspace, shard) +func (tee *Tee) GetShard(ctx context.Context, keyspace, shard string) (*pb.Shard, int64, error) { + s, v, err := tee.readFrom.GetShard(ctx, keyspace, shard) if err != nil { - return nil, err + return nil, 0, err } - si2, err := tee.readFromSecond.GetShard(ctx, keyspace, shard) + _, v2, err := tee.readFromSecond.GetShard(ctx, keyspace, shard) if err != nil { // can't read from secondary, so we can's keep version map - return si, nil + return s, v, nil } tee.mu.Lock() tee.shardVersionMapping[keyspace+"/"+shard] = versionMapping{ - readFromVersion: si.Version(), - readFromSecondVersion: si2.Version(), + readFromVersion: v, + readFromSecondVersion: v2, } tee.mu.Unlock() - return si, nil + return s, v, nil } // GetShardNames is part of the topo.Server interface diff --git a/go/vt/topo/server.go b/go/vt/topo/server.go index 5ad31b05c4..fce3bb9e6e 100644 --- a/go/vt/topo/server.go +++ b/go/vt/topo/server.go @@ -120,9 +120,9 @@ type Impl interface { // ValidateShard performs routine checks on the shard. ValidateShard(ctx context.Context, keyspace, shard string) error - // GetShard reads a shard and returns it. + // GetShard reads a shard and returns it, along with its version. // Can return ErrNoNode - GetShard(ctx context.Context, keyspace, shard string) (*ShardInfo, error) + GetShard(ctx context.Context, keyspace, shard string) (*pb.Shard, int64, error) // GetShardNames returns the known shards in a keyspace. // Can return ErrNoNode if the keyspace wasn't created, diff --git a/go/vt/topo/shard.go b/go/vt/topo/shard.go index c5ed70317e..03e09cfe2a 100644 --- a/go/vt/topo/shard.go +++ b/go/vt/topo/shard.go @@ -200,7 +200,16 @@ func (ts Server) GetShard(ctx context.Context, keyspace, shard string) (*ShardIn span.Annotate("shard", shard) defer span.Finish() - return ts.Impl.GetShard(ctx, keyspace, shard) + value, version, err := ts.Impl.GetShard(ctx, keyspace, shard) + if err != nil { + return nil, err + } + return &ShardInfo{ + keyspace: keyspace, + shardName: shard, + version: version, + Shard: value, + }, nil } // UpdateShard updates the shard data, with the right version. diff --git a/go/vt/topo/test/faketopo/faketopo.go b/go/vt/topo/test/faketopo/faketopo.go index 2ca6b10fbc..a7280444ac 100644 --- a/go/vt/topo/test/faketopo/faketopo.go +++ b/go/vt/topo/test/faketopo/faketopo.go @@ -85,8 +85,8 @@ func (ft FakeTopo) ValidateShard(ctx context.Context, keyspace, shard string) er } // GetShard implements topo.Server. -func (ft FakeTopo) GetShard(ctx context.Context, keyspace, shard string) (*topo.ShardInfo, error) { - return nil, errNotImplemented +func (ft FakeTopo) GetShard(ctx context.Context, keyspace, shard string) (*pb.Shard, int64, error) { + return nil, 0, errNotImplemented } // GetShardNames implements topo.Server. diff --git a/go/vt/topo/test/shard.go b/go/vt/topo/test/shard.go index 8b6d8fbe87..c475ca0e9a 100644 --- a/go/vt/topo/test/shard.go +++ b/go/vt/topo/test/shard.go @@ -59,21 +59,21 @@ func CheckShard(ctx context.Context, t *testing.T, ts topo.Impl) { t.Fatalf("CreateShard: %v", err) } - if _, err := ts.GetShard(ctx, "test_keyspace", "666"); err != topo.ErrNoNode { + if _, _, err := ts.GetShard(ctx, "test_keyspace", "666"); err != topo.ErrNoNode { t.Errorf("GetShard(666): %v", err) } - shardInfo, err := ts.GetShard(ctx, "test_keyspace", "b0-c0") + shard, version, err := ts.GetShard(ctx, "test_keyspace", "b0-c0") if err != nil { t.Errorf("GetShard: %v", err) } - if want := newKeyRange3("b0-c0"); !key.KeyRangeEqual(shardInfo.KeyRange, want) { - t.Errorf("shardInfo.KeyRange: want %v, got %v", want, shardInfo.KeyRange) + if want := newKeyRange3("b0-c0"); !key.KeyRangeEqual(shard.KeyRange, want) { + t.Errorf("shard.KeyRange: want %v, got %v", want, shard.KeyRange) } master := &pb.TabletAlias{Cell: "ny", Uid: 1} - shardInfo.MasterAlias = master - shardInfo.KeyRange = newKeyRange3("b0-c0") - shardInfo.ServedTypes = []*pb.Shard_ServedType{ + shard.MasterAlias = master + shard.KeyRange = newKeyRange3("b0-c0") + shard.ServedTypes = []*pb.Shard_ServedType{ &pb.Shard_ServedType{ TabletType: pb.TabletType_MASTER, }, @@ -85,7 +85,7 @@ func CheckShard(ctx context.Context, t *testing.T, ts topo.Impl) { TabletType: pb.TabletType_RDONLY, }, } - shardInfo.SourceShards = []*pb.Shard_SourceShard{ + shard.SourceShards = []*pb.Shard_SourceShard{ &pb.Shard_SourceShard{ Uid: 1, Keyspace: "source_ks", @@ -94,7 +94,7 @@ func CheckShard(ctx context.Context, t *testing.T, ts topo.Impl) { Tables: []string{"table1", "table2"}, }, } - shardInfo.TabletControls = []*pb.Shard_TabletControl{ + shard.TabletControls = []*pb.Shard_TabletControl{ &pb.Shard_TabletControl{ TabletType: pb.TabletType_MASTER, Cells: []string{"c1", "c2"}, @@ -105,7 +105,7 @@ func CheckShard(ctx context.Context, t *testing.T, ts topo.Impl) { DisableQueryService: true, }, } - if err := tts.UpdateShard(ctx, shardInfo); err != nil { + if _, err := ts.UpdateShard(ctx, "test_keyspace", "b0-c0", shard, version); err != nil { t.Errorf("UpdateShard: %v", err) } @@ -117,12 +117,12 @@ func CheckShard(ctx context.Context, t *testing.T, ts topo.Impl) { if err != nil { t.Fatalf("UpdateShardFields error: %v", err) } - si, err := ts.GetShard(ctx, "test_keyspace", "b0-c0") + s, _, err := ts.GetShard(ctx, "test_keyspace", "b0-c0") if err != nil { t.Fatalf("GetShard: %v", err) } - if *si.MasterAlias != *other { - t.Fatalf("shard.MasterAlias = %v, want %v", si.MasterAlias, other) + if *s.MasterAlias != *other { + t.Fatalf("shard.MasterAlias = %v, want %v", s.MasterAlias, other) } _, err = topo.UpdateShardFields(ctx, tts, "test_keyspace", "b0-c0", func(shard *pb.Shard) error { shard.MasterAlias = master @@ -132,15 +132,15 @@ func CheckShard(ctx context.Context, t *testing.T, ts topo.Impl) { t.Fatalf("UpdateShardFields error: %v", err) } - updatedShardInfo, err := ts.GetShard(ctx, "test_keyspace", "b0-c0") + updatedShard, _, err := ts.GetShard(ctx, "test_keyspace", "b0-c0") if err != nil { t.Fatalf("GetShard: %v", err) } - if eq, err := shardEqual(shardInfo.Shard, updatedShardInfo.Shard); err != nil { + if eq, err := shardEqual(shard, updatedShard); err != nil { t.Errorf("cannot compare shards: %v", err) } else if !eq { - t.Errorf("put and got shards are not identical:\n%#v\n%#v", shardInfo.Shard, updatedShardInfo.Shard) + t.Errorf("put and got shards are not identical:\n%#v\n%#v", shard, updatedShard) } // test GetShardNames diff --git a/go/vt/zktopo/shard.go b/go/vt/zktopo/shard.go index e9acddb0f4..e7b07f40c1 100644 --- a/go/vt/zktopo/shard.go +++ b/go/vt/zktopo/shard.go @@ -100,22 +100,22 @@ func (zkts *Server) ValidateShard(ctx context.Context, keyspace, shard string) e } // GetShard is part of the topo.Server interface -func (zkts *Server) GetShard(ctx context.Context, keyspace, shard string) (*topo.ShardInfo, error) { +func (zkts *Server) GetShard(ctx context.Context, keyspace, shard string) (*pb.Shard, int64, error) { shardPath := path.Join(globalKeyspacesPath, keyspace, "shards", shard) data, stat, err := zkts.zconn.Get(shardPath) if err != nil { if zookeeper.IsError(err, zookeeper.ZNONODE) { err = topo.ErrNoNode } - return nil, err + return nil, 0, err } s := &pb.Shard{} if err = json.Unmarshal([]byte(data), s); err != nil { - return nil, fmt.Errorf("bad shard data %v", err) + return nil, 0, fmt.Errorf("bad shard data %v", err) } - return topo.NewShardInfo(keyspace, shard, s, int64(stat.Version())), nil + return s, int64(stat.Version()), nil } // GetShardNames is part of the topo.Server interface From 1ca1a79a74b827bb97cbd884bf0aed7f0aba8426 Mon Sep 17 00:00:00 2001 From: Alain Jobart Date: Thu, 20 Aug 2015 08:50:45 -0700 Subject: [PATCH 72/80] Now fixing CreateShard. --- go/cmd/vtctld/api_test.go | 4 +-- go/cmd/vtctld/topo_data_test.go | 8 +++--- go/vt/etcdtopo/shard.go | 7 ----- go/vt/tabletmanager/healthcheck_test.go | 2 +- go/vt/topo/helpers/copy.go | 2 +- go/vt/topo/shard.go | 18 ++++++++++--- go/vt/topo/test/lock.go | 4 ++- go/vt/topo/test/shard.go | 11 +++++--- go/vt/topotools/shard.go | 4 +-- go/vt/worker/split_clone_test.go | 3 +-- go/vt/worker/split_diff_test.go | 27 +++++++++---------- go/vt/wrangler/rebuild.go | 2 +- go/vt/wrangler/testlib/reparent_utils_test.go | 5 ++-- go/vt/zktopo/shard.go | 7 ----- 14 files changed, 51 insertions(+), 53 deletions(-) diff --git a/go/cmd/vtctld/api_test.go b/go/cmd/vtctld/api_test.go index d5b3397689..a654aaeda9 100644 --- a/go/cmd/vtctld/api_test.go +++ b/go/cmd/vtctld/api_test.go @@ -42,11 +42,11 @@ func TestAPI(t *testing.T) { // Populate topo. ts.CreateKeyspace(ctx, "ks1", &pb.Keyspace{ShardingColumnName: "shardcol"}) - ts.CreateShard(ctx, "ks1", "-80", &pb.Shard{ + ts.Impl.CreateShard(ctx, "ks1", "-80", &pb.Shard{ Cells: cells, KeyRange: &pb.KeyRange{Start: nil, End: []byte{0x80}}, }) - ts.CreateShard(ctx, "ks1", "80-", &pb.Shard{ + ts.Impl.CreateShard(ctx, "ks1", "80-", &pb.Shard{ Cells: cells, KeyRange: &pb.KeyRange{Start: []byte{0x80}, End: nil}, }) diff --git a/go/cmd/vtctld/topo_data_test.go b/go/cmd/vtctld/topo_data_test.go index 7f5fb1f7fc..4736fa0c41 100644 --- a/go/cmd/vtctld/topo_data_test.go +++ b/go/cmd/vtctld/topo_data_test.go @@ -225,12 +225,12 @@ func TestShardNamesCache(t *testing.T) { }); err != nil { t.Fatalf("CreateKeyspace failed: %v", err) } - if err := ts.CreateShard(ctx, "ks1", "s1", &pb.Shard{ + if err := ts.Impl.CreateShard(ctx, "ks1", "s1", &pb.Shard{ Cells: []string{"cell1", "cell2"}, }); err != nil { t.Fatalf("CreateShard failed: %v", err) } - if err := ts.CreateShard(ctx, "ks1", "s2", &pb.Shard{ + if err := ts.Impl.CreateShard(ctx, "ks1", "s2", &pb.Shard{ MasterAlias: &pb.TabletAlias{ Cell: "cell1", Uid: 12, @@ -256,12 +256,12 @@ func TestShardCache(t *testing.T) { }); err != nil { t.Fatalf("CreateKeyspace failed: %v", err) } - if err := ts.CreateShard(ctx, "ks1", "s1", &pb.Shard{ + if err := ts.Impl.CreateShard(ctx, "ks1", "s1", &pb.Shard{ Cells: []string{"cell1", "cell2"}, }); err != nil { t.Fatalf("CreateShard failed: %v", err) } - if err := ts.CreateShard(ctx, "ks1", "s2", &pb.Shard{ + if err := ts.Impl.CreateShard(ctx, "ks1", "s2", &pb.Shard{ MasterAlias: &pb.TabletAlias{ Cell: "cell1", Uid: 12, diff --git a/go/vt/etcdtopo/shard.go b/go/vt/etcdtopo/shard.go index cb09538dcc..b38189ad7a 100644 --- a/go/vt/etcdtopo/shard.go +++ b/go/vt/etcdtopo/shard.go @@ -29,13 +29,6 @@ func (s *Server) CreateShard(ctx context.Context, keyspace, shard string, value if err = initLockFile(global, shardDirPath(keyspace, shard)); err != nil { return err } - - event.Dispatch(&events.ShardChange{ - KeyspaceName: keyspace, - ShardName: shard, - Shard: value, - Status: "created", - }) return nil } diff --git a/go/vt/tabletmanager/healthcheck_test.go b/go/vt/tabletmanager/healthcheck_test.go index 3f2548eaa5..ec171ae094 100644 --- a/go/vt/tabletmanager/healthcheck_test.go +++ b/go/vt/tabletmanager/healthcheck_test.go @@ -120,7 +120,7 @@ func createTestAgent(ctx context.Context, t *testing.T) *ActionAgent { t.Fatalf("CreateKeyspace failed: %v", err) } - if err := topo.CreateShard(ctx, ts, keyspace, shard); err != nil { + if err := ts.CreateShard(ctx, keyspace, shard); err != nil { t.Fatalf("CreateShard failed: %v", err) } diff --git a/go/vt/topo/helpers/copy.go b/go/vt/topo/helpers/copy.go index 05ca21a00a..8824be72b7 100644 --- a/go/vt/topo/helpers/copy.go +++ b/go/vt/topo/helpers/copy.go @@ -83,7 +83,7 @@ func CopyShards(ctx context.Context, fromTS, toTS topo.Impl, deleteKeyspaceShard wg.Add(1) go func(keyspace, shard string) { defer wg.Done() - if err := topo.CreateShard(ctx, topo.Server{Impl: toTS}, keyspace, shard); err != nil { + if err := toTS.CreateShard(ctx, keyspace, shard, &pb.Shard{}); err != nil { if err == topo.ErrNodeExists { log.Warningf("shard %v/%v already exists", keyspace, shard) } else { diff --git a/go/vt/topo/shard.go b/go/vt/topo/shard.go index 03e09cfe2a..2316e29a0a 100644 --- a/go/vt/topo/shard.go +++ b/go/vt/topo/shard.go @@ -264,7 +264,7 @@ func UpdateShardFields(ctx context.Context, ts Server, keyspace, shard string, u // This should be called while holding the keyspace lock for the shard. // (call topotools.CreateShard to do that for you). // In unit tests (that are not parallel), this function can be called directly. -func CreateShard(ctx context.Context, ts Server, keyspace, shard string) error { +func (ts Server) CreateShard(ctx context.Context, keyspace, shard string) error { name, keyRange, err := ValidateShardName(shard) if err != nil { return err @@ -277,7 +277,7 @@ func CreateShard(ctx context.Context, ts Server, keyspace, shard string) error { pb.TabletType_REPLICA: true, pb.TabletType_RDONLY: true, } - s := &pb.Shard{ + value := &pb.Shard{ KeyRange: keyRange, } @@ -298,12 +298,22 @@ func CreateShard(ctx context.Context, ts Server, keyspace, shard string) error { } for st := range servedTypes { - s.ServedTypes = append(s.ServedTypes, &pb.Shard_ServedType{ + value.ServedTypes = append(value.ServedTypes, &pb.Shard_ServedType{ TabletType: st, }) } - return ts.CreateShard(ctx, keyspace, name, s) + if err := ts.Impl.CreateShard(ctx, keyspace, name, value); err != nil { + return err + } + + event.Dispatch(&events.ShardChange{ + KeyspaceName: keyspace, + ShardName: shard, + Shard: value, + Status: "created", + }) + return nil } // GetTabletControl returns the Shard_TabletControl for the given tablet type, diff --git a/go/vt/topo/test/lock.go b/go/vt/topo/test/lock.go index 0be6b2ee32..416f4132d0 100644 --- a/go/vt/topo/test/lock.go +++ b/go/vt/topo/test/lock.go @@ -118,7 +118,9 @@ func CheckShardLock(ctx context.Context, t *testing.T, ts topo.Impl) { if err := ts.CreateKeyspace(ctx, "test_keyspace", &pb.Keyspace{}); err != nil { t.Fatalf("CreateKeyspace: %v", err) } - if err := topo.CreateShard(ctx, topo.Server{Impl: ts}, "test_keyspace", "10-20"); err != nil { + if err := ts.CreateShard(ctx, "test_keyspace", "10-20", &pb.Shard{ + KeyRange: newKeyRange3("10-20"), + }); err != nil { t.Fatalf("CreateShard: %v", err) } diff --git a/go/vt/topo/test/shard.go b/go/vt/topo/test/shard.go index c475ca0e9a..76fd5d5432 100644 --- a/go/vt/topo/test/shard.go +++ b/go/vt/topo/test/shard.go @@ -36,10 +36,13 @@ func CheckShard(ctx context.Context, t *testing.T, ts topo.Impl) { t.Fatalf("CreateKeyspace: %v", err) } - if err := topo.CreateShard(ctx, tts, "test_keyspace", "b0-c0"); err != nil { + shard := &pb.Shard{ + KeyRange: newKeyRange3("b0-c0"), + } + if err := ts.CreateShard(ctx, "test_keyspace", "b0-c0", shard); err != nil { t.Fatalf("CreateShard: %v", err) } - if err := topo.CreateShard(ctx, tts, "test_keyspace", "b0-c0"); err != topo.ErrNodeExists { + if err := ts.CreateShard(ctx, "test_keyspace", "b0-c0", shard); err != topo.ErrNodeExists { t.Errorf("CreateShard called second time, got: %v", err) } @@ -47,7 +50,7 @@ func CheckShard(ctx context.Context, t *testing.T, ts topo.Impl) { if err := ts.DeleteShard(ctx, "test_keyspace", "b0-c0"); err != nil { t.Fatalf("DeleteShard: %v", err) } - if err := topo.CreateShard(ctx, tts, "test_keyspace", "b0-c0"); err != nil { + if err := ts.CreateShard(ctx, "test_keyspace", "b0-c0", shard); err != nil { t.Fatalf("CreateShard: %v", err) } @@ -55,7 +58,7 @@ func CheckShard(ctx context.Context, t *testing.T, ts topo.Impl) { if err := ts.DeleteKeyspaceShards(ctx, "test_keyspace"); err != nil { t.Fatalf("DeleteKeyspaceShards: %v", err) } - if err := topo.CreateShard(ctx, tts, "test_keyspace", "b0-c0"); err != nil { + if err := ts.CreateShard(ctx, "test_keyspace", "b0-c0", shard); err != nil { t.Fatalf("CreateShard: %v", err) } diff --git a/go/vt/topotools/shard.go b/go/vt/topotools/shard.go index b0cc5857f5..ba5707ab22 100644 --- a/go/vt/topotools/shard.go +++ b/go/vt/topotools/shard.go @@ -24,7 +24,7 @@ func CreateShard(ctx context.Context, ts topo.Server, keyspace, shard string) er } // now try to create within the lock, may already exist - err = topo.CreateShard(ctx, ts, keyspace, shard) + err = ts.CreateShard(ctx, keyspace, shard) // and unlock and return return node.UnlockKeyspace(ctx, ts, keyspace, lockPath, err) @@ -48,7 +48,7 @@ func GetOrCreateShard(ctx context.Context, ts topo.Server, keyspace, shard strin } // now try to create within the lock, may already exist - if err := topo.CreateShard(ctx, ts, keyspace, shard); err != nil && err != topo.ErrNodeExists { + if err := ts.CreateShard(ctx, keyspace, shard); err != nil && err != topo.ErrNodeExists { return nil, node.UnlockKeyspace(ctx, ts, keyspace, lockPath, fmt.Errorf("CreateShard(%v/%v) failed: %v", keyspace, shard, err)) } diff --git a/go/vt/worker/split_clone_test.go b/go/vt/worker/split_clone_test.go index d222bb460e..9171f76cdb 100644 --- a/go/vt/worker/split_clone_test.go +++ b/go/vt/worker/split_clone_test.go @@ -21,7 +21,6 @@ import ( "github.com/youtube/vitess/go/vt/tabletserver/grpcqueryservice" "github.com/youtube/vitess/go/vt/tabletserver/proto" "github.com/youtube/vitess/go/vt/tabletserver/queryservice" - "github.com/youtube/vitess/go/vt/topo" "github.com/youtube/vitess/go/vt/wrangler" "github.com/youtube/vitess/go/vt/wrangler/testlib" "github.com/youtube/vitess/go/vt/zktopo" @@ -271,7 +270,7 @@ func testSplitClone(t *testing.T, strategy string) { // add the topo and schema data we'll need ctx := context.Background() - if err := topo.CreateShard(ctx, ts, "ks", "80-"); err != nil { + if err := ts.CreateShard(ctx, "ks", "80-"); err != nil { t.Fatalf("CreateShard(\"-80\") failed: %v", err) } if err := wr.SetKeyspaceShardingInfo(ctx, "ks", "keyspace_id", pbt.KeyspaceIdType_UINT64, 4, false); err != nil { diff --git a/go/vt/worker/split_diff_test.go b/go/vt/worker/split_diff_test.go index 7dbcbdc6cd..0fe2826fb7 100644 --- a/go/vt/worker/split_diff_test.go +++ b/go/vt/worker/split_diff_test.go @@ -18,7 +18,6 @@ import ( "github.com/youtube/vitess/go/vt/tabletserver/grpcqueryservice" "github.com/youtube/vitess/go/vt/tabletserver/proto" "github.com/youtube/vitess/go/vt/tabletserver/queryservice" - "github.com/youtube/vitess/go/vt/topo" "github.com/youtube/vitess/go/vt/wrangler" "github.com/youtube/vitess/go/vt/wrangler/testlib" "github.com/youtube/vitess/go/vt/zktopo" @@ -28,15 +27,15 @@ import ( pbt "github.com/youtube/vitess/go/vt/proto/topodata" ) -// destinationSqlQuery is a local QueryService implementation to +// destinationSQLQuery is a local QueryService implementation to // support the tests -type destinationSqlQuery struct { +type destinationSQLQuery struct { queryservice.ErrorQueryService t *testing.T excludedTable string } -func (sq *destinationSqlQuery) StreamExecute(ctx context.Context, target *pb.Target, query *proto.Query, sendReply func(reply *mproto.QueryResult) error) error { +func (sq *destinationSQLQuery) StreamExecute(ctx context.Context, target *pb.Target, query *proto.Query, sendReply func(reply *mproto.QueryResult) error) error { if strings.Contains(query.Sql, sq.excludedTable) { sq.t.Errorf("Split Diff operation on destination should skip the excluded table: %v query: %v", sq.excludedTable, query.Sql) } @@ -45,7 +44,7 @@ func (sq *destinationSqlQuery) StreamExecute(ctx context.Context, target *pb.Tar sq.t.Errorf("Sql query on destination should not contain a keyspace_id WHERE clause; query received: %v", query.Sql) } - sq.t.Logf("destinationSqlQuery: got query: %v", *query) + sq.t.Logf("destinationSQLQuery: got query: %v", *query) // Send the headers if err := sendReply(&mproto.QueryResult{ @@ -85,14 +84,14 @@ func (sq *destinationSqlQuery) StreamExecute(ctx context.Context, target *pb.Tar return nil } -// sourceSqlQuery is a local QueryService implementation to support the tests -type sourceSqlQuery struct { +// sourceSQLQuery is a local QueryService implementation to support the tests +type sourceSQLQuery struct { queryservice.ErrorQueryService t *testing.T excludedTable string } -func (sq *sourceSqlQuery) StreamExecute(ctx context.Context, target *pb.Target, query *proto.Query, sendReply func(reply *mproto.QueryResult) error) error { +func (sq *sourceSQLQuery) StreamExecute(ctx context.Context, target *pb.Target, query *proto.Query, sendReply func(reply *mproto.QueryResult) error) error { if strings.Contains(query.Sql, sq.excludedTable) { sq.t.Errorf("Split Diff operation on source should skip the excluded table: %v query: %v", sq.excludedTable, query.Sql) } @@ -104,7 +103,7 @@ func (sq *sourceSqlQuery) StreamExecute(ctx context.Context, target *pb.Target, } } - sq.t.Logf("sourceSqlQuery: got query: %v", *query) + sq.t.Logf("sourceSQLQuery: got query: %v", *query) // Send the headers if err := sendReply(&mproto.QueryResult{ @@ -173,7 +172,7 @@ func TestSplitDiff(t *testing.T) { } // add the topo and schema data we'll need - if err := topo.CreateShard(ctx, ts, "ks", "80-"); err != nil { + if err := ts.CreateShard(ctx, "ks", "80-"); err != nil { t.Fatalf("CreateShard(\"-80\") failed: %v", err) } wr.SetSourceShards(ctx, "ks", "-40", []*pbt.TabletAlias{sourceRdonly1.Tablet.Alias}, nil) @@ -216,10 +215,10 @@ func TestSplitDiff(t *testing.T) { } } - grpcqueryservice.RegisterForTest(leftRdonly1.RPCServer, &destinationSqlQuery{t: t, excludedTable: excludedTable}) - grpcqueryservice.RegisterForTest(leftRdonly2.RPCServer, &destinationSqlQuery{t: t, excludedTable: excludedTable}) - grpcqueryservice.RegisterForTest(sourceRdonly1.RPCServer, &sourceSqlQuery{t: t, excludedTable: excludedTable}) - grpcqueryservice.RegisterForTest(sourceRdonly2.RPCServer, &sourceSqlQuery{t: t, excludedTable: excludedTable}) + grpcqueryservice.RegisterForTest(leftRdonly1.RPCServer, &destinationSQLQuery{t: t, excludedTable: excludedTable}) + grpcqueryservice.RegisterForTest(leftRdonly2.RPCServer, &destinationSQLQuery{t: t, excludedTable: excludedTable}) + grpcqueryservice.RegisterForTest(sourceRdonly1.RPCServer, &sourceSQLQuery{t: t, excludedTable: excludedTable}) + grpcqueryservice.RegisterForTest(sourceRdonly2.RPCServer, &sourceSQLQuery{t: t, excludedTable: excludedTable}) err := wrk.Run(ctx) status := wrk.StatusAsText() diff --git a/go/vt/wrangler/rebuild.go b/go/vt/wrangler/rebuild.go index 66537add92..3abbacbefa 100644 --- a/go/vt/wrangler/rebuild.go +++ b/go/vt/wrangler/rebuild.go @@ -249,7 +249,7 @@ func (wr *Wrangler) RebuildReplicationGraph(ctx context.Context, cells []string, keyspacesToRebuild[ti.Keyspace] = true shardPath := ti.Keyspace + "/" + ti.Shard if !shardsCreated[shardPath] { - if err := topo.CreateShard(ctx, wr.ts, ti.Keyspace, ti.Shard); err != nil && err != topo.ErrNodeExists { + if err := wr.ts.CreateShard(ctx, ti.Keyspace, ti.Shard); err != nil && err != topo.ErrNodeExists { wr.logger.Warningf("failed re-creating shard %v: %v", shardPath, err) hasErr = true } else { diff --git a/go/vt/wrangler/testlib/reparent_utils_test.go b/go/vt/wrangler/testlib/reparent_utils_test.go index cf8f9cf706..82edc915a7 100644 --- a/go/vt/wrangler/testlib/reparent_utils_test.go +++ b/go/vt/wrangler/testlib/reparent_utils_test.go @@ -12,7 +12,6 @@ import ( "github.com/youtube/vitess/go/vt/logutil" myproto "github.com/youtube/vitess/go/vt/mysqlctl/proto" "github.com/youtube/vitess/go/vt/tabletmanager/tmclient" - "github.com/youtube/vitess/go/vt/topo" "github.com/youtube/vitess/go/vt/topo/topoproto" "github.com/youtube/vitess/go/vt/wrangler" "github.com/youtube/vitess/go/vt/zktopo" @@ -27,7 +26,7 @@ func TestShardReplicationStatuses(t *testing.T) { wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), time.Second) // create shard and tablets - if err := topo.CreateShard(ctx, ts, "test_keyspace", "0"); err != nil { + if err := ts.CreateShard(ctx, "test_keyspace", "0"); err != nil { t.Fatalf("CreateShard failed: %v", err) } master := NewFakeTablet(t, wr, "cell1", 1, pb.TabletType_MASTER) @@ -95,7 +94,7 @@ func TestReparentTablet(t *testing.T) { wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), time.Second) // create shard and tablets - if err := topo.CreateShard(ctx, ts, "test_keyspace", "0"); err != nil { + if err := ts.CreateShard(ctx, "test_keyspace", "0"); err != nil { t.Fatalf("CreateShard failed: %v", err) } master := NewFakeTablet(t, wr, "cell1", 1, pb.TabletType_MASTER) diff --git a/go/vt/zktopo/shard.go b/go/vt/zktopo/shard.go index e7b07f40c1..aaa9296d18 100644 --- a/go/vt/zktopo/shard.go +++ b/go/vt/zktopo/shard.go @@ -56,13 +56,6 @@ func (zkts *Server) CreateShard(ctx context.Context, keyspace, shard string, val if alreadyExists { return topo.ErrNodeExists } - - event.Dispatch(&events.ShardChange{ - KeyspaceName: keyspace, - ShardName: shard, - Shard: value, - Status: "created", - }) return nil } From c9c0cbdaf8487b5e753e7699024e1d8d822b9867 Mon Sep 17 00:00:00 2001 From: Alain Jobart Date: Thu, 20 Aug 2015 09:13:33 -0700 Subject: [PATCH 73/80] Now wrapping DeleteShard. --- go/vt/etcdtopo/shard.go | 9 --------- go/vt/topo/shard.go | 15 +++++++++++++++ go/vt/zktopo/shard.go | 9 --------- 3 files changed, 15 insertions(+), 18 deletions(-) diff --git a/go/vt/etcdtopo/shard.go b/go/vt/etcdtopo/shard.go index b38189ad7a..e411d659ca 100644 --- a/go/vt/etcdtopo/shard.go +++ b/go/vt/etcdtopo/shard.go @@ -8,8 +8,6 @@ import ( "encoding/json" "fmt" - "github.com/youtube/vitess/go/event" - "github.com/youtube/vitess/go/vt/topo/events" "golang.org/x/net/context" pb "github.com/youtube/vitess/go/vt/proto/topodata" @@ -89,12 +87,5 @@ func (s *Server) DeleteShard(ctx context.Context, keyspace, shard string) error if err != nil { return convertError(err) } - - event.Dispatch(&events.ShardChange{ - KeyspaceName: keyspace, - ShardName: shard, - Shard: nil, - Status: "deleted", - }) return nil } diff --git a/go/vt/topo/shard.go b/go/vt/topo/shard.go index 2316e29a0a..eb66d0f122 100644 --- a/go/vt/topo/shard.go +++ b/go/vt/topo/shard.go @@ -316,6 +316,21 @@ func (ts Server) CreateShard(ctx context.Context, keyspace, shard string) error return nil } +// DeleteShard wraps the underlying Impl.DeleteShard +// and dispatches the event. +func (ts Server) DeleteShard(ctx context.Context, keyspace, shard string) error { + if err := ts.Impl.DeleteShard(ctx, keyspace, shard); err != nil { + return err + } + event.Dispatch(&events.ShardChange{ + KeyspaceName: keyspace, + ShardName: shard, + Shard: nil, + Status: "deleted", + }) + return nil +} + // GetTabletControl returns the Shard_TabletControl for the given tablet type, // or nil if it is not in the map. func (si *ShardInfo) GetTabletControl(tabletType pb.TabletType) *pb.Shard_TabletControl { diff --git a/go/vt/zktopo/shard.go b/go/vt/zktopo/shard.go index aaa9296d18..5bb58223be 100644 --- a/go/vt/zktopo/shard.go +++ b/go/vt/zktopo/shard.go @@ -10,9 +10,7 @@ import ( "path" "sort" - "github.com/youtube/vitess/go/event" "github.com/youtube/vitess/go/vt/topo" - "github.com/youtube/vitess/go/vt/topo/events" "github.com/youtube/vitess/go/zk" "golang.org/x/net/context" "launchpad.net/gozk/zookeeper" @@ -136,12 +134,5 @@ func (zkts *Server) DeleteShard(ctx context.Context, keyspace, shard string) err } return err } - - event.Dispatch(&events.ShardChange{ - KeyspaceName: keyspace, - ShardName: shard, - Shard: nil, - Status: "deleted", - }) return nil } From a6f771fd00cf622482660824336913b73aa12f59 Mon Sep 17 00:00:00 2001 From: Alain Jobart Date: Thu, 20 Aug 2015 09:31:03 -0700 Subject: [PATCH 74/80] Movng a few more methods to topo.Server. --- go/cmd/vtctld/api.go | 4 ++-- go/vt/tabletmanager/reparent.go | 2 +- go/vt/topo/shard.go | 24 ++++++++----------- go/vt/topo/tablet.go | 2 +- go/vt/topo/test/shard.go | 4 ++-- go/vt/topotools/rebuild.go | 2 +- go/vt/topotools/utils.go | 2 +- go/vt/vtctl/vtctl.go | 4 ++-- go/vt/worker/clone_utils.go | 4 ++-- go/vt/wrangler/keyspace.go | 2 +- go/vt/wrangler/permissions.go | 5 ++-- go/vt/wrangler/reparent.go | 8 +++---- go/vt/wrangler/schema.go | 8 +++---- go/vt/wrangler/shard.go | 2 +- go/vt/wrangler/split.go | 3 +-- .../testlib/reparent_external_test.go | 6 ++--- go/vt/wrangler/validator.go | 6 ++--- go/vt/wrangler/version.go | 5 ++-- go/vt/wrangler/zkns.go | 3 +-- 19 files changed, 44 insertions(+), 52 deletions(-) diff --git a/go/cmd/vtctld/api.go b/go/cmd/vtctld/api.go index cc43bfaa8d..414aba3cfd 100644 --- a/go/cmd/vtctld/api.go +++ b/go/cmd/vtctld/api.go @@ -164,9 +164,9 @@ func initAPI(ctx context.Context, ts topo.Server, actions *ActionRepository) { return nil, err } if cell != "" { - return topo.FindAllTabletAliasesInShardByCell(ctx, ts, keyspace, shard, []string{cell}) + return ts.FindAllTabletAliasesInShardByCell(ctx, keyspace, shard, []string{cell}) } - return topo.FindAllTabletAliasesInShard(ctx, ts, keyspace, shard) + return ts.FindAllTabletAliasesInShard(ctx, keyspace, shard) } // Get all tablets in a cell. diff --git a/go/vt/tabletmanager/reparent.go b/go/vt/tabletmanager/reparent.go index 7ac9cc9332..329bf57546 100644 --- a/go/vt/tabletmanager/reparent.go +++ b/go/vt/tabletmanager/reparent.go @@ -221,7 +221,7 @@ func (agent *ActionAgent) finalizeTabletExternallyReparented(ctx context.Context // write it back. Now we use an update loop pattern to do that instead. event.DispatchUpdate(ev, "updating global shard record") log.Infof("finalizeTabletExternallyReparented: updating global shard record") - si, err = topo.UpdateShardFields(ctx, agent.TopoServer, tablet.Keyspace, tablet.Shard, func(shard *pb.Shard) error { + si, err = agent.TopoServer.UpdateShardFields(ctx, tablet.Keyspace, tablet.Shard, func(shard *pb.Shard) error { shard.MasterAlias = tablet.Alias return nil }) diff --git a/go/vt/topo/shard.go b/go/vt/topo/shard.go index eb66d0f122..9466490d62 100644 --- a/go/vt/topo/shard.go +++ b/go/vt/topo/shard.go @@ -77,12 +77,8 @@ func removeCells(cells, toRemove, fullList []string) ([]string, bool) { } // ParseKeyspaceShardString parse a "keyspace/shard" string and extract -// both keyspace and shard. It also returns empty keyspace and shard if -// input param looks like a old zk path +// both keyspace and shard func ParseKeyspaceShardString(param string) (string, string, error) { - if param[0] == '/' { - return "", "", fmt.Errorf("Invalid keyspace/shard: %v, Note: old style zk path is no longer supported, please use a keyspace/shard instead", param) - } keySpaceShard := strings.Split(param, "/") if len(keySpaceShard) != 2 { return "", "", fmt.Errorf("Invalid shard path: %v", param) @@ -245,7 +241,7 @@ func (ts Server) UpdateShard(ctx context.Context, si *ShardInfo) error { // update function on it, and then write it back. If the write fails due to // a version mismatch, it will re-read the record and retry the update. // If the update succeeds, it returns the updated ShardInfo. -func UpdateShardFields(ctx context.Context, ts Server, keyspace, shard string, update func(*pb.Shard) error) (*ShardInfo, error) { +func (ts Server) UpdateShardFields(ctx context.Context, keyspace, shard string, update func(*pb.Shard) error) (*ShardInfo, error) { for { si, err := ts.GetShard(ctx, keyspace, shard) if err != nil { @@ -552,8 +548,8 @@ func InCellList(cell string, cells []string) bool { // in which case the result only contains the cells that were fetched. // // The tablet aliases are sorted by cell, then by UID. -func FindAllTabletAliasesInShard(ctx context.Context, ts Server, keyspace, shard string) ([]*pb.TabletAlias, error) { - return FindAllTabletAliasesInShardByCell(ctx, ts, keyspace, shard, nil) +func (ts Server) FindAllTabletAliasesInShard(ctx context.Context, keyspace, shard string) ([]*pb.TabletAlias, error) { + return ts.FindAllTabletAliasesInShardByCell(ctx, keyspace, shard, nil) } // FindAllTabletAliasesInShardByCell uses the replication graph to find all the @@ -563,7 +559,7 @@ func FindAllTabletAliasesInShard(ctx context.Context, ts Server, keyspace, shard // in which case the result only contains the cells that were fetched. // // The tablet aliases are sorted by cell, then by UID. -func FindAllTabletAliasesInShardByCell(ctx context.Context, ts Server, keyspace, shard string, cells []string) ([]*pb.TabletAlias, error) { +func (ts Server) FindAllTabletAliasesInShardByCell(ctx context.Context, keyspace, shard string, cells []string) ([]*pb.TabletAlias, error) { span := trace.NewSpanFromContext(ctx) span.StartLocal("topo.FindAllTabletAliasesInShardbyCell") span.Annotate("keyspace", keyspace) @@ -628,24 +624,24 @@ func FindAllTabletAliasesInShardByCell(ctx context.Context, ts Server, keyspace, // GetTabletMapForShard returns the tablets for a shard. It can return // ErrPartialResult if it couldn't read all the cells, or all // the individual tablets, in which case the map is valid, but partial. -func GetTabletMapForShard(ctx context.Context, ts Server, keyspace, shard string) (map[pb.TabletAlias]*TabletInfo, error) { - return GetTabletMapForShardByCell(ctx, ts, keyspace, shard, nil) +func (ts Server) GetTabletMapForShard(ctx context.Context, keyspace, shard string) (map[pb.TabletAlias]*TabletInfo, error) { + return ts.GetTabletMapForShardByCell(ctx, keyspace, shard, nil) } // GetTabletMapForShardByCell returns the tablets for a shard. It can return // ErrPartialResult if it couldn't read all the cells, or all // the individual tablets, in which case the map is valid, but partial. -func GetTabletMapForShardByCell(ctx context.Context, ts Server, keyspace, shard string, cells []string) (map[pb.TabletAlias]*TabletInfo, error) { +func (ts Server) GetTabletMapForShardByCell(ctx context.Context, keyspace, shard string, cells []string) (map[pb.TabletAlias]*TabletInfo, error) { // if we get a partial result, we keep going. It most likely means // a cell is out of commission. - aliases, err := FindAllTabletAliasesInShardByCell(ctx, ts, keyspace, shard, cells) + aliases, err := ts.FindAllTabletAliasesInShardByCell(ctx, keyspace, shard, cells) if err != nil && err != ErrPartialResult { return nil, err } // get the tablets for the cells we were able to reach, forward // ErrPartialResult from FindAllTabletAliasesInShard - result, gerr := GetTabletMap(ctx, ts, aliases) + result, gerr := ts.GetTabletMap(ctx, aliases) if gerr == nil && err != nil { gerr = err } diff --git a/go/vt/topo/tablet.go b/go/vt/topo/tablet.go index 8e2b8073dc..8e2483a84a 100644 --- a/go/vt/topo/tablet.go +++ b/go/vt/topo/tablet.go @@ -410,7 +410,7 @@ func DeleteTabletReplicationData(ctx context.Context, ts Server, tablet *pb.Tabl // and returns them all in a map. // If error is ErrPartialResult, the results in the dictionary are // incomplete, meaning some tablets couldn't be read. -func GetTabletMap(ctx context.Context, ts Server, tabletAliases []*pb.TabletAlias) (map[pb.TabletAlias]*TabletInfo, error) { +func (ts Server) GetTabletMap(ctx context.Context, tabletAliases []*pb.TabletAlias) (map[pb.TabletAlias]*TabletInfo, error) { span := trace.NewSpanFromContext(ctx) span.StartLocal("topo.GetTabletMap") span.Annotate("num_tablets", len(tabletAliases)) diff --git a/go/vt/topo/test/shard.go b/go/vt/topo/test/shard.go index 76fd5d5432..cac4692ec1 100644 --- a/go/vt/topo/test/shard.go +++ b/go/vt/topo/test/shard.go @@ -113,7 +113,7 @@ func CheckShard(ctx context.Context, t *testing.T, ts topo.Impl) { } other := &pb.TabletAlias{Cell: "ny", Uid: 82873} - _, err = topo.UpdateShardFields(ctx, tts, "test_keyspace", "b0-c0", func(shard *pb.Shard) error { + _, err = tts.UpdateShardFields(ctx, "test_keyspace", "b0-c0", func(shard *pb.Shard) error { shard.MasterAlias = other return nil }) @@ -127,7 +127,7 @@ func CheckShard(ctx context.Context, t *testing.T, ts topo.Impl) { if *s.MasterAlias != *other { t.Fatalf("shard.MasterAlias = %v, want %v", s.MasterAlias, other) } - _, err = topo.UpdateShardFields(ctx, tts, "test_keyspace", "b0-c0", func(shard *pb.Shard) error { + _, err = tts.UpdateShardFields(ctx, "test_keyspace", "b0-c0", func(shard *pb.Shard) error { shard.MasterAlias = master return nil }) diff --git a/go/vt/topotools/rebuild.go b/go/vt/topotools/rebuild.go index 4ec28cf392..f386b50bf8 100644 --- a/go/vt/topotools/rebuild.go +++ b/go/vt/topotools/rebuild.go @@ -76,7 +76,7 @@ func rebuildCellSrvShard(ctx context.Context, log logutil.Logger, ts topo.Server versions, err := getEndPointsVersions(ctx, ts, cell, si.Keyspace(), si.ShardName()) // Get all tablets in this cell/shard. - tablets, err := topo.GetTabletMapForShardByCell(ctx, ts, si.Keyspace(), si.ShardName(), []string{cell}) + tablets, err := ts.GetTabletMapForShardByCell(ctx, si.Keyspace(), si.ShardName(), []string{cell}) if err != nil { if err != topo.ErrPartialResult { return err diff --git a/go/vt/topotools/utils.go b/go/vt/topotools/utils.go index b63fc24719..1e34f3c786 100644 --- a/go/vt/topotools/utils.go +++ b/go/vt/topotools/utils.go @@ -36,7 +36,7 @@ func GetAllTablets(ctx context.Context, ts topo.Server, cell string) ([]*topo.Ta } sort.Sort(topoproto.TabletAliasList(aliases)) - tabletMap, err := topo.GetTabletMap(ctx, ts, aliases) + tabletMap, err := ts.GetTabletMap(ctx, aliases) if err != nil { // we got another error than topo.ErrNoNode return nil, err diff --git a/go/vt/vtctl/vtctl.go b/go/vt/vtctl/vtctl.go index f130820ffc..a5e4709106 100644 --- a/go/vt/vtctl/vtctl.go +++ b/go/vt/vtctl/vtctl.go @@ -435,7 +435,7 @@ func fmtAction(action *actionnode.ActionNode) string { } func listTabletsByShard(ctx context.Context, wr *wrangler.Wrangler, keyspace, shard string) error { - tabletAliases, err := topo.FindAllTabletAliasesInShard(ctx, wr.TopoServer(), keyspace, shard) + tabletAliases, err := wr.TopoServer().FindAllTabletAliasesInShard(ctx, keyspace, shard) if err != nil { return err } @@ -454,7 +454,7 @@ func dumpAllTablets(ctx context.Context, wr *wrangler.Wrangler, zkVtPath string) } func dumpTablets(ctx context.Context, wr *wrangler.Wrangler, tabletAliases []*pb.TabletAlias) error { - tabletMap, err := topo.GetTabletMap(ctx, wr.TopoServer(), tabletAliases) + tabletMap, err := wr.TopoServer().GetTabletMap(ctx, tabletAliases) if err != nil { return err } diff --git a/go/vt/worker/clone_utils.go b/go/vt/worker/clone_utils.go index 658688888a..7c60e6f836 100644 --- a/go/vt/worker/clone_utils.go +++ b/go/vt/worker/clone_utils.go @@ -62,7 +62,7 @@ func resolveDestinationShardMaster(ctx context.Context, keyspace, shard string, func resolveReloadTabletsForShard(ctx context.Context, keyspace, shard string, wr *wrangler.Wrangler) (reloadAliases []*pb.TabletAlias, reloadTablets map[pb.TabletAlias]*topo.TabletInfo, err error) { // Keep a long timeout, because we really don't want the copying to succeed, and then the worker to fail at the end. shortCtx, cancel := context.WithTimeout(ctx, 5*time.Minute) - reloadAliases, err = topo.FindAllTabletAliasesInShard(shortCtx, wr.TopoServer(), keyspace, shard) + reloadAliases, err = wr.TopoServer().FindAllTabletAliasesInShard(shortCtx, keyspace, shard) cancel() if err != nil { return nil, nil, fmt.Errorf("cannot find all reload target tablets in %v/%v: %v", keyspace, shard, err) @@ -70,7 +70,7 @@ func resolveReloadTabletsForShard(ctx context.Context, keyspace, shard string, w wr.Logger().Infof("Found %v reload target aliases in shard %v/%v", len(reloadAliases), keyspace, shard) shortCtx, cancel = context.WithTimeout(ctx, 5*time.Minute) - reloadTablets, err = topo.GetTabletMap(shortCtx, wr.TopoServer(), reloadAliases) + reloadTablets, err = wr.TopoServer().GetTabletMap(shortCtx, reloadAliases) cancel() if err != nil { return nil, nil, fmt.Errorf("cannot read all reload target tablets in %v/%v: %v", diff --git a/go/vt/wrangler/keyspace.go b/go/vt/wrangler/keyspace.go index 38a46e524b..49ccf12e4a 100644 --- a/go/vt/wrangler/keyspace.go +++ b/go/vt/wrangler/keyspace.go @@ -703,7 +703,7 @@ func (wr *Wrangler) setKeyspaceServedFrom(ctx context.Context, keyspace string, // discovery wouldn't be very efficient. func (wr *Wrangler) RefreshTablesByShard(ctx context.Context, si *topo.ShardInfo, tabletType pb.TabletType, cells []string) error { wr.Logger().Infof("RefreshTablesByShard called on shard %v/%v", si.Keyspace(), si.ShardName()) - tabletMap, err := topo.GetTabletMapForShardByCell(ctx, wr.ts, si.Keyspace(), si.ShardName(), cells) + tabletMap, err := wr.ts.GetTabletMapForShardByCell(ctx, si.Keyspace(), si.ShardName(), cells) switch err { case nil: // keep going diff --git a/go/vt/wrangler/permissions.go b/go/vt/wrangler/permissions.go index 7d86d7f605..3c51f86e11 100644 --- a/go/vt/wrangler/permissions.go +++ b/go/vt/wrangler/permissions.go @@ -12,7 +12,6 @@ import ( log "github.com/golang/glog" "github.com/youtube/vitess/go/vt/concurrency" myproto "github.com/youtube/vitess/go/vt/mysqlctl/proto" - "github.com/youtube/vitess/go/vt/topo" "github.com/youtube/vitess/go/vt/topo/topoproto" "golang.org/x/net/context" @@ -63,7 +62,7 @@ func (wr *Wrangler) ValidatePermissionsShard(ctx context.Context, keyspace, shar // read all the aliases in the shard, that is all tablets that are // replicating from the master - aliases, err := topo.FindAllTabletAliasesInShard(ctx, wr.ts, keyspace, shard) + aliases, err := wr.ts.FindAllTabletAliasesInShard(ctx, keyspace, shard) if err != nil { return err } @@ -122,7 +121,7 @@ func (wr *Wrangler) ValidatePermissionsKeyspace(ctx context.Context, keyspace st er := concurrency.AllErrorRecorder{} wg := sync.WaitGroup{} for _, shard := range shards { - aliases, err := topo.FindAllTabletAliasesInShard(ctx, wr.ts, keyspace, shard) + aliases, err := wr.ts.FindAllTabletAliasesInShard(ctx, keyspace, shard) if err != nil { er.RecordError(err) continue diff --git a/go/vt/wrangler/reparent.go b/go/vt/wrangler/reparent.go index a63f3aa77a..f2833ac216 100644 --- a/go/vt/wrangler/reparent.go +++ b/go/vt/wrangler/reparent.go @@ -62,7 +62,7 @@ func (wr *Wrangler) ShardReplicationStatuses(ctx context.Context, keyspace, shar func (wr *Wrangler) shardReplicationStatuses(ctx context.Context, shardInfo *topo.ShardInfo) ([]*topo.TabletInfo, []*myproto.ReplicationStatus, error) { // FIXME(msolomon) this assumes no hierarchical replication, which is currently the case. - tabletMap, err := topo.GetTabletMapForShard(ctx, wr.ts, shardInfo.Keyspace(), shardInfo.ShardName()) + tabletMap, err := wr.ts.GetTabletMapForShard(ctx, shardInfo.Keyspace(), shardInfo.ShardName()) if err != nil { return nil, nil, err } @@ -180,7 +180,7 @@ func (wr *Wrangler) initShardMasterLocked(ctx context.Context, ev *events.Repare ev.ShardInfo = *shardInfo event.DispatchUpdate(ev, "reading tablet map") - tabletMap, err := topo.GetTabletMapForShard(ctx, wr.ts, keyspace, shard) + tabletMap, err := wr.ts.GetTabletMapForShard(ctx, keyspace, shard) if err != nil { return err } @@ -345,7 +345,7 @@ func (wr *Wrangler) plannedReparentShardLocked(ctx context.Context, ev *events.R ev.ShardInfo = *shardInfo event.DispatchUpdate(ev, "reading tablet map") - tabletMap, err := topo.GetTabletMapForShard(ctx, wr.ts, keyspace, shard) + tabletMap, err := wr.ts.GetTabletMapForShard(ctx, keyspace, shard) if err != nil { return err } @@ -477,7 +477,7 @@ func (wr *Wrangler) emergencyReparentShardLocked(ctx context.Context, ev *events ev.ShardInfo = *shardInfo event.DispatchUpdate(ev, "reading all tablets") - tabletMap, err := topo.GetTabletMapForShard(ctx, wr.ts, keyspace, shard) + tabletMap, err := wr.ts.GetTabletMapForShard(ctx, keyspace, shard) if err != nil { return err } diff --git a/go/vt/wrangler/schema.go b/go/vt/wrangler/schema.go index e85ca0e9bd..b1aa7bca86 100644 --- a/go/vt/wrangler/schema.go +++ b/go/vt/wrangler/schema.go @@ -80,7 +80,7 @@ func (wr *Wrangler) ValidateSchemaShard(ctx context.Context, keyspace, shard str // read all the aliases in the shard, that is all tablets that are // replicating from the master - aliases, err := topo.FindAllTabletAliasesInShard(ctx, wr.ts, keyspace, shard) + aliases, err := wr.ts.FindAllTabletAliasesInShard(ctx, keyspace, shard) if err != nil { return err } @@ -141,7 +141,7 @@ func (wr *Wrangler) ValidateSchemaKeyspace(ctx context.Context, keyspace string, wg := sync.WaitGroup{} // first diff the slaves in the reference shard 0 - aliases, err := topo.FindAllTabletAliasesInShard(ctx, wr.ts, keyspace, shards[0]) + aliases, err := wr.ts.FindAllTabletAliasesInShard(ctx, keyspace, shards[0]) if err != nil { return err } @@ -168,7 +168,7 @@ func (wr *Wrangler) ValidateSchemaKeyspace(ctx context.Context, keyspace string, continue } - aliases, err := topo.FindAllTabletAliasesInShard(ctx, wr.ts, keyspace, shard) + aliases, err := wr.ts.FindAllTabletAliasesInShard(ctx, keyspace, shard) if err != nil { er.RecordError(err) continue @@ -256,7 +256,7 @@ type tabletStatus struct { func (wr *Wrangler) applySchemaShard(ctx context.Context, shardInfo *topo.ShardInfo, preflight *myproto.SchemaChangeResult, masterTabletAlias *pb.TabletAlias, change string, newParentTabletAlias *pb.TabletAlias, simple, force bool, waitSlaveTimeout time.Duration) (*myproto.SchemaChangeResult, error) { // find all the shards we need to handle - aliases, err := topo.FindAllTabletAliasesInShard(ctx, wr.ts, shardInfo.Keyspace(), shardInfo.ShardName()) + aliases, err := wr.ts.FindAllTabletAliasesInShard(ctx, shardInfo.Keyspace(), shardInfo.ShardName()) if err != nil { return nil, err } diff --git a/go/vt/wrangler/shard.go b/go/vt/wrangler/shard.go index e323dc62bf..e2fcea7e06 100644 --- a/go/vt/wrangler/shard.go +++ b/go/vt/wrangler/shard.go @@ -159,7 +159,7 @@ func (wr *Wrangler) DeleteShard(ctx context.Context, keyspace, shard string, rec return err } - tabletMap, err := topo.GetTabletMapForShard(ctx, wr.ts, keyspace, shard) + tabletMap, err := wr.ts.GetTabletMapForShard(ctx, keyspace, shard) if err != nil { return err } diff --git a/go/vt/wrangler/split.go b/go/vt/wrangler/split.go index b6c0712b0c..a955b4e6b9 100644 --- a/go/vt/wrangler/split.go +++ b/go/vt/wrangler/split.go @@ -7,7 +7,6 @@ package wrangler import ( "fmt" - "github.com/youtube/vitess/go/vt/topo" "golang.org/x/net/context" pb "github.com/youtube/vitess/go/vt/proto/topodata" @@ -29,7 +28,7 @@ func (wr *Wrangler) SetSourceShards(ctx context.Context, keyspace, shard string, } // read the source tablets - sourceTablets, err := topo.GetTabletMap(ctx, wr.TopoServer(), sources) + sourceTablets, err := wr.ts.GetTabletMap(ctx, sources) if err != nil { return err } diff --git a/go/vt/wrangler/testlib/reparent_external_test.go b/go/vt/wrangler/testlib/reparent_external_test.go index 4a07da5d4c..b17429dbc2 100644 --- a/go/vt/wrangler/testlib/reparent_external_test.go +++ b/go/vt/wrangler/testlib/reparent_external_test.go @@ -55,7 +55,7 @@ func TestTabletExternallyReparented(t *testing.T) { // Slightly unrelated test: make sure we can find the tablets // even with a datacenter being down. - tabletMap, err := topo.GetTabletMapForShardByCell(ctx, ts, "test_keyspace", "0", []string{"cell1"}) + tabletMap, err := ts.GetTabletMapForShardByCell(ctx, "test_keyspace", "0", []string{"cell1"}) if err != nil { t.Fatalf("GetTabletMapForShardByCell should have worked but got: %v", err) } @@ -73,13 +73,13 @@ func TestTabletExternallyReparented(t *testing.T) { } // Make sure the master is not exported in other cells - tabletMap, err = topo.GetTabletMapForShardByCell(ctx, ts, "test_keyspace", "0", []string{"cell2"}) + tabletMap, err = ts.GetTabletMapForShardByCell(ctx, "test_keyspace", "0", []string{"cell2"}) master, err = topotools.FindTabletByIPAddrAndPort(tabletMap, oldMaster.Tablet.Ip, "vt", oldMaster.Tablet.PortMap["vt"]) if err != topo.ErrNoNode { t.Fatalf("FindTabletByIPAddrAndPort(master) worked in cell2: %v %v", err, master) } - tabletMap, err = topo.GetTabletMapForShard(ctx, ts, "test_keyspace", "0") + tabletMap, err = ts.GetTabletMapForShard(ctx, "test_keyspace", "0") if err != topo.ErrPartialResult { t.Fatalf("GetTabletMapForShard should have returned ErrPartialResult but got: %v", err) } diff --git a/go/vt/wrangler/validator.go b/go/vt/wrangler/validator.go index 5aa8e4fbe0..8272116508 100644 --- a/go/vt/wrangler/validator.go +++ b/go/vt/wrangler/validator.go @@ -60,7 +60,7 @@ func (wr *Wrangler) validateAllTablets(ctx context.Context, wg *sync.WaitGroup, } for _, shard := range shards { - aliases, err := topo.FindAllTabletAliasesInShard(ctx, wr.ts, keyspace, shard) + aliases, err := wr.ts.FindAllTabletAliasesInShard(ctx, keyspace, shard) if err != nil { results <- fmt.Errorf("TopologyServer.FindAllTabletAliasesInShard(%v, %v) failed: %v", keyspace, shard, err) return @@ -117,13 +117,13 @@ func (wr *Wrangler) validateShard(ctx context.Context, keyspace, shard string, p return } - aliases, err := topo.FindAllTabletAliasesInShard(ctx, wr.ts, keyspace, shard) + aliases, err := wr.ts.FindAllTabletAliasesInShard(ctx, keyspace, shard) if err != nil { results <- fmt.Errorf("TopologyServer.FindAllTabletAliasesInShard(%v, %v) failed: %v", keyspace, shard, err) return } - tabletMap, _ := topo.GetTabletMap(ctx, wr.ts, aliases) + tabletMap, _ := wr.ts.GetTabletMap(ctx, aliases) var masterAlias *pb.TabletAlias for _, alias := range aliases { diff --git a/go/vt/wrangler/version.go b/go/vt/wrangler/version.go index 023e146d52..38efa772c3 100644 --- a/go/vt/wrangler/version.go +++ b/go/vt/wrangler/version.go @@ -14,7 +14,6 @@ import ( log "github.com/golang/glog" "github.com/youtube/vitess/go/vt/concurrency" - "github.com/youtube/vitess/go/vt/topo" "github.com/youtube/vitess/go/vt/topo/topoproto" "golang.org/x/net/context" @@ -107,7 +106,7 @@ func (wr *Wrangler) ValidateVersionShard(ctx context.Context, keyspace, shard st // read all the aliases in the shard, that is all tablets that are // replicating from the master - aliases, err := topo.FindAllTabletAliasesInShard(ctx, wr.ts, keyspace, shard) + aliases, err := wr.ts.FindAllTabletAliasesInShard(ctx, keyspace, shard) if err != nil { return err } @@ -167,7 +166,7 @@ func (wr *Wrangler) ValidateVersionKeyspace(ctx context.Context, keyspace string er := concurrency.AllErrorRecorder{} wg := sync.WaitGroup{} for _, shard := range shards { - aliases, err := topo.FindAllTabletAliasesInShard(ctx, wr.ts, keyspace, shard) + aliases, err := wr.ts.FindAllTabletAliasesInShard(ctx, keyspace, shard) if err != nil { er.RecordError(err) continue diff --git a/go/vt/wrangler/zkns.go b/go/vt/wrangler/zkns.go index 9ad5618975..2cd3dbc5e2 100644 --- a/go/vt/wrangler/zkns.go +++ b/go/vt/wrangler/zkns.go @@ -8,7 +8,6 @@ import ( "strings" log "github.com/golang/glog" - "github.com/youtube/vitess/go/vt/topo" "github.com/youtube/vitess/go/vt/topo/topoproto" "github.com/youtube/vitess/go/vt/zktopo" "github.com/youtube/vitess/go/zk" @@ -67,7 +66,7 @@ func (wr *Wrangler) ExportZknsForKeyspace(ctx context.Context, keyspace string) } // Scan the first shard to discover which cells need local serving data. - aliases, err := topo.FindAllTabletAliasesInShard(ctx, wr.ts, keyspace, shardNames[0]) + aliases, err := wr.ts.FindAllTabletAliasesInShard(ctx, keyspace, shardNames[0]) if err != nil { return err } From 55a80fb912e121b8e2ee22b2694d925ca75fb847 Mon Sep 17 00:00:00 2001 From: Alain Jobart Date: Thu, 20 Aug 2015 09:50:11 -0700 Subject: [PATCH 75/80] Moving proto-related shard methods to topo/topoproto/. --- go/cmd/vtctld/api.go | 2 +- go/cmd/vtctld/topo_data.go | 3 +- go/vt/tabletmanager/binlog.go | 4 +- go/vt/topo/shard.go | 31 -------------- go/vt/topo/shard_test.go | 23 ----------- go/vt/topo/topoproto/shard.go | 45 ++++++++++++++++++++ go/vt/topo/topoproto/shard_test.go | 52 ++++++++++++++++++++++++ go/vt/vtctl/backup.go | 6 +-- go/vt/vtctl/reparent.go | 7 ++-- go/vt/vtctl/vtctl.go | 52 ++++++++++++------------ go/vt/worker/split_clone_cmd.go | 4 +- go/vt/worker/split_diff_cmd.go | 4 +- go/vt/worker/vertical_split_clone_cmd.go | 4 +- go/vt/worker/vertical_split_diff_cmd.go | 4 +- 14 files changed, 142 insertions(+), 99 deletions(-) create mode 100644 go/vt/topo/topoproto/shard.go create mode 100644 go/vt/topo/topoproto/shard_test.go diff --git a/go/cmd/vtctld/api.go b/go/cmd/vtctld/api.go index 414aba3cfd..d5a1a4f0c0 100644 --- a/go/cmd/vtctld/api.go +++ b/go/cmd/vtctld/api.go @@ -159,7 +159,7 @@ func initAPI(ctx context.Context, ts topo.Server, actions *ActionRepository) { if shardRef != "" { // Look up by keyspace/shard, and optionally cell. - keyspace, shard, err := topo.ParseKeyspaceShardString(shardRef) + keyspace, shard, err := topoproto.ParseKeyspaceShard(shardRef) if err != nil { return nil, err } diff --git a/go/cmd/vtctld/topo_data.go b/go/cmd/vtctld/topo_data.go index 4c7d819f41..fb03c9d41f 100644 --- a/go/cmd/vtctld/topo_data.go +++ b/go/cmd/vtctld/topo_data.go @@ -9,6 +9,7 @@ import ( "time" "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "golang.org/x/net/context" pb "github.com/youtube/vitess/go/vt/proto/topodata" @@ -306,7 +307,7 @@ func newShardCache(ts topo.Server) *VersionedObjectCacheMap { return NewVersionedObjectCacheMap(func(key string) *VersionedObjectCache { return NewVersionedObjectCache(func(ctx context.Context) (VersionedObject, error) { - keyspace, shard, err := topo.ParseKeyspaceShardString(key) + keyspace, shard, err := topoproto.ParseKeyspaceShard(key) if err != nil { return nil, err } diff --git a/go/vt/tabletmanager/binlog.go b/go/vt/tabletmanager/binlog.go index d93e08e08b..18f4ad7fcd 100644 --- a/go/vt/tabletmanager/binlog.go +++ b/go/vt/tabletmanager/binlog.go @@ -94,7 +94,7 @@ func newBinlogPlayerController(ts topo.Server, dbConfig *sqldb.ConnParams, mysql } func (bpc *BinlogPlayerController) String() string { - return "BinlogPlayerController(" + topo.SourceShardString(bpc.sourceShard) + ")" + return "BinlogPlayerController(" + topoproto.SourceShardString(bpc.sourceShard) + ")" } // Start will start the player in the background and run forever. @@ -599,7 +599,7 @@ type BinlogPlayerControllerStatus struct { // SourceShardAsHTML returns the SourceShard as HTML func (bpcs *BinlogPlayerControllerStatus) SourceShardAsHTML() template.HTML { - return topo.SourceShardAsHTML(bpcs.SourceShard) + return topoproto.SourceShardAsHTML(bpcs.SourceShard) } // SourceTabletAlias returns the string version of the SourceTablet alias, if set diff --git a/go/vt/topo/shard.go b/go/vt/topo/shard.go index 9466490d62..44f8e84a39 100644 --- a/go/vt/topo/shard.go +++ b/go/vt/topo/shard.go @@ -7,7 +7,6 @@ package topo import ( "encoding/hex" "fmt" - "html/template" "reflect" "sort" "strings" @@ -76,36 +75,6 @@ func removeCells(cells, toRemove, fullList []string) ([]string, bool) { return leftoverCells, false } -// ParseKeyspaceShardString parse a "keyspace/shard" string and extract -// both keyspace and shard -func ParseKeyspaceShardString(param string) (string, string, error) { - keySpaceShard := strings.Split(param, "/") - if len(keySpaceShard) != 2 { - return "", "", fmt.Errorf("Invalid shard path: %v", param) - } - return keySpaceShard[0], keySpaceShard[1], nil -} - -// SourceShardString returns a printable view of a SourceShard. -func SourceShardString(source *pb.Shard_SourceShard) string { - return fmt.Sprintf("SourceShard(%v,%v/%v)", source.Uid, source.Keyspace, source.Shard) -} - -// SourceShardAsHTML returns a HTML version of the object. -func SourceShardAsHTML(source *pb.Shard_SourceShard) template.HTML { - result := fmt.Sprintf("Uid: %v
\nSource: %v/%v
\n", source.Uid, source.Keyspace, source.Shard) - if key.KeyRangeIsPartial(source.KeyRange) { - result += fmt.Sprintf("KeyRange: %v-%v
\n", - hex.EncodeToString(source.KeyRange.Start), - hex.EncodeToString(source.KeyRange.End)) - } - if len(source.Tables) > 0 { - result += fmt.Sprintf("Tables: %v
\n", - strings.Join(source.Tables, " ")) - } - return template.HTML(result) -} - // IsShardUsingRangeBasedSharding returns true if the shard name // implies it is using range based sharding. func IsShardUsingRangeBasedSharding(shard string) bool { diff --git a/go/vt/topo/shard_test.go b/go/vt/topo/shard_test.go index 495adca6ef..eff353f8cb 100644 --- a/go/vt/topo/shard_test.go +++ b/go/vt/topo/shard_test.go @@ -61,29 +61,6 @@ func TestRemoveCells(t *testing.T) { } } -func TestParseKeyspaceShardString(t *testing.T) { - zkPath := "/zk/tablet" - keyspace := "key01" - shard := "shard0" - tabletAlias := keyspace + "/" + shard - - if _, _, err := ParseKeyspaceShardString(zkPath); err == nil { - t.Fatalf("zk path: %s should cause error.", zkPath) - } - k, s, err := ParseKeyspaceShardString(tabletAlias) - if err != nil { - t.Fatalf("Failed to parse valid tablet alias: %s", tabletAlias) - } - if keyspace != k { - t.Fatalf("keyspace parsed from tablet alias %s is %s, but expect %s", - tabletAlias, k, keyspace) - } - if shard != s { - t.Fatalf("shard parsed from tablet alias %s is %s, but expect %s", - tabletAlias, s, shard) - } -} - func TestUpdateSourceBlacklistedTables(t *testing.T) { si := NewShardInfo("ks", "sh", &pb.Shard{ Cells: []string{"first", "second", "third"}, diff --git a/go/vt/topo/topoproto/shard.go b/go/vt/topo/topoproto/shard.go new file mode 100644 index 0000000000..872a1dbdbd --- /dev/null +++ b/go/vt/topo/topoproto/shard.go @@ -0,0 +1,45 @@ +// Copyright 2015, Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package topoproto + +import ( + "encoding/hex" + "fmt" + "html/template" + "strings" + + "github.com/youtube/vitess/go/vt/key" + pb "github.com/youtube/vitess/go/vt/proto/topodata" +) + +// ParseKeyspaceShard parse a "keyspace/shard" string and extract +// both keyspace and shard +func ParseKeyspaceShard(param string) (string, string, error) { + keySpaceShard := strings.Split(param, "/") + if len(keySpaceShard) != 2 { + return "", "", fmt.Errorf("Invalid shard path: %v", param) + } + return keySpaceShard[0], keySpaceShard[1], nil +} + +// SourceShardString returns a printable view of a SourceShard. +func SourceShardString(source *pb.Shard_SourceShard) string { + return fmt.Sprintf("SourceShard(%v,%v/%v)", source.Uid, source.Keyspace, source.Shard) +} + +// SourceShardAsHTML returns a HTML version of the object. +func SourceShardAsHTML(source *pb.Shard_SourceShard) template.HTML { + result := fmt.Sprintf("Uid: %v
\nSource: %v/%v
\n", source.Uid, source.Keyspace, source.Shard) + if key.KeyRangeIsPartial(source.KeyRange) { + result += fmt.Sprintf("KeyRange: %v-%v
\n", + hex.EncodeToString(source.KeyRange.Start), + hex.EncodeToString(source.KeyRange.End)) + } + if len(source.Tables) > 0 { + result += fmt.Sprintf("Tables: %v
\n", + strings.Join(source.Tables, " ")) + } + return template.HTML(result) +} diff --git a/go/vt/topo/topoproto/shard_test.go b/go/vt/topo/topoproto/shard_test.go new file mode 100644 index 0000000000..a8e509ef8b --- /dev/null +++ b/go/vt/topo/topoproto/shard_test.go @@ -0,0 +1,52 @@ +// Copyright 2015, Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package topoproto + +import ( + "testing" + + pb "github.com/youtube/vitess/go/vt/proto/topodata" +) + +func TestParseKeyspaceShard(t *testing.T) { + zkPath := "/zk/global/ks/sh" + keyspace := "key01" + shard := "shard0" + keyspaceShard := keyspace + "/" + shard + + if _, _, err := ParseKeyspaceShard(zkPath); err == nil { + t.Errorf("zk path: %s should cause error.", zkPath) + } + k, s, err := ParseKeyspaceShard(keyspaceShard) + if err != nil { + t.Fatalf("Failed to parse valid keyspace/shard pair: %s", keyspaceShard) + } + if k != keyspace { + t.Errorf("keyspace parsed from keyspace/shard pair %s is %s, but expect %s", keyspaceShard, k, keyspace) + } + if s != shard { + t.Errorf("shard parsed from keyspace/shard pair %s is %s, but expect %s", keyspaceShard, s, shard) + } +} + +func TestSourceShardAsHTML(t *testing.T) { + s := &pb.Shard_SourceShard{ + Uid: 123, + Keyspace: "source_keyspace", + Shard: "source_shard", + KeyRange: &pb.KeyRange{ + Start: []byte{0x80}, + }, + Tables: []string{"table1", "table2"}, + } + got := string(SourceShardAsHTML(s)) + expected := "Uid: 123
\n" + + "Source: source_keyspace/source_shard
\n" + + "KeyRange: 80-
\n" + + "Tables: table1 table2
\n" + if got != expected { + t.Errorf("got wrong SourceShardAsHTML output, got:\n%vexpected:\n%v", got, expected) + } +} diff --git a/go/vt/vtctl/backup.go b/go/vt/vtctl/backup.go index eaa7e77604..0050bbd992 100644 --- a/go/vt/vtctl/backup.go +++ b/go/vt/vtctl/backup.go @@ -9,7 +9,7 @@ import ( "fmt" "github.com/youtube/vitess/go/vt/mysqlctl/backupstorage" - "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "github.com/youtube/vitess/go/vt/wrangler" "golang.org/x/net/context" ) @@ -35,7 +35,7 @@ func commandListBackups(ctx context.Context, wr *wrangler.Wrangler, subFlags *fl return fmt.Errorf("action ListBackups requires ") } - keyspace, shard, err := topo.ParseKeyspaceShardString(subFlags.Arg(0)) + keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(0)) if err != nil { return err } @@ -63,7 +63,7 @@ func commandRemoveBackup(ctx context.Context, wr *wrangler.Wrangler, subFlags *f return fmt.Errorf("action RemoveBackup requires ") } - keyspace, shard, err := topo.ParseKeyspaceShardString(subFlags.Arg(0)) + keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(0)) if err != nil { return err } diff --git a/go/vt/vtctl/reparent.go b/go/vt/vtctl/reparent.go index 8c94bd7bcb..795a5bfb9d 100644 --- a/go/vt/vtctl/reparent.go +++ b/go/vt/vtctl/reparent.go @@ -9,7 +9,6 @@ import ( "fmt" "time" - "github.com/youtube/vitess/go/vt/topo" "github.com/youtube/vitess/go/vt/topo/topoproto" "github.com/youtube/vitess/go/vt/wrangler" "golang.org/x/net/context" @@ -101,7 +100,7 @@ func commandInitShardMaster(ctx context.Context, wr *wrangler.Wrangler, subFlags if subFlags.NArg() != 2 { return fmt.Errorf("action InitShardMaster requires ") } - keyspace, shard, err := topo.ParseKeyspaceShardString(subFlags.Arg(0)) + keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(0)) if err != nil { return err } @@ -125,7 +124,7 @@ func commandPlannedReparentShard(ctx context.Context, wr *wrangler.Wrangler, sub return fmt.Errorf("action PlannedReparentShard requires ") } - keyspace, shard, err := topo.ParseKeyspaceShardString(subFlags.Arg(0)) + keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(0)) if err != nil { return err } @@ -149,7 +148,7 @@ func commandEmergencyReparentShard(ctx context.Context, wr *wrangler.Wrangler, s return fmt.Errorf("action EmergencyReparentShard requires ") } - keyspace, shard, err := topo.ParseKeyspaceShardString(subFlags.Arg(0)) + keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(0)) if err != nil { return err } diff --git a/go/vt/vtctl/vtctl.go b/go/vt/vtctl/vtctl.go index a5e4709106..273fcfebe9 100644 --- a/go/vt/vtctl/vtctl.go +++ b/go/vt/vtctl/vtctl.go @@ -526,7 +526,7 @@ func shardParamsToKeyspaceShards(ctx context.Context, wr *wrangler.Wrangler, par if param[0] == '/' { // this is a topology-specific path for _, path := range params { - keyspace, shard, err := topo.ParseKeyspaceShardString(path) + keyspace, shard, err := topoproto.ParseKeyspaceShard(path) if err != nil { return nil, err } @@ -1027,7 +1027,7 @@ func commandCreateShard(ctx context.Context, wr *wrangler.Wrangler, subFlags *fl return fmt.Errorf("The argument is required for the CreateShard command.") } - keyspace, shard, err := topo.ParseKeyspaceShardString(subFlags.Arg(0)) + keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(0)) if err != nil { return err } @@ -1053,7 +1053,7 @@ func commandGetShard(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag. return fmt.Errorf("The argument is required for the GetShard command.") } - keyspace, shard, err := topo.ParseKeyspaceShardString(subFlags.Arg(0)) + keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(0)) if err != nil { return err } @@ -1118,7 +1118,7 @@ func commandValidateShard(ctx context.Context, wr *wrangler.Wrangler, subFlags * return fmt.Errorf("The argument is required for the ValidateShard command.") } - keyspace, shard, err := topo.ParseKeyspaceShardString(subFlags.Arg(0)) + keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(0)) if err != nil { return err } @@ -1132,7 +1132,7 @@ func commandShardReplicationPositions(ctx context.Context, wr *wrangler.Wrangler if subFlags.NArg() != 1 { return fmt.Errorf("The argument is required for the ShardReplicationPositions command.") } - keyspace, shard, err := topo.ParseKeyspaceShardString(subFlags.Arg(0)) + keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(0)) if err != nil { return err } @@ -1164,7 +1164,7 @@ func commandListShardTablets(ctx context.Context, wr *wrangler.Wrangler, subFlag if subFlags.NArg() != 1 { return fmt.Errorf("The argument is required for the ListShardTablets command.") } - keyspace, shard, err := topo.ParseKeyspaceShardString(subFlags.Arg(0)) + keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(0)) if err != nil { return err } @@ -1180,7 +1180,7 @@ func commandSetShardServedTypes(ctx context.Context, wr *wrangler.Wrangler, subF if subFlags.NArg() != 2 { return fmt.Errorf("The and arguments are both required for the SetShardServedTypes command.") } - keyspace, shard, err := topo.ParseKeyspaceShardString(subFlags.Arg(0)) + keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(0)) if err != nil { return err } @@ -1208,7 +1208,7 @@ func commandSetShardTabletControl(ctx context.Context, wr *wrangler.Wrangler, su if subFlags.NArg() != 2 { return fmt.Errorf("The and arguments are both required for the SetShardTabletControl command.") } - keyspace, shard, err := topo.ParseKeyspaceShardString(subFlags.Arg(0)) + keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(0)) if err != nil { return err } @@ -1236,7 +1236,7 @@ func commandSourceShardDelete(ctx context.Context, wr *wrangler.Wrangler, subFla if subFlags.NArg() < 2 { return fmt.Errorf("The and arguments are both required for the SourceShardDelete command.") } - keyspace, shard, err := topo.ParseKeyspaceShardString(subFlags.Arg(0)) + keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(0)) if err != nil { return err } @@ -1256,7 +1256,7 @@ func commandSourceShardAdd(ctx context.Context, wr *wrangler.Wrangler, subFlags if subFlags.NArg() != 3 { return fmt.Errorf("The , , and arguments are all required for the SourceShardAdd command.") } - keyspace, shard, err := topo.ParseKeyspaceShardString(subFlags.Arg(0)) + keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(0)) if err != nil { return err } @@ -1264,7 +1264,7 @@ func commandSourceShardAdd(ctx context.Context, wr *wrangler.Wrangler, subFlags if err != nil { return err } - skeyspace, sshard, err := topo.ParseKeyspaceShardString(subFlags.Arg(2)) + skeyspace, sshard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(2)) if err != nil { return err } @@ -1289,7 +1289,7 @@ func commandShardReplicationAdd(ctx context.Context, wr *wrangler.Wrangler, subF return fmt.Errorf("The and arguments are required for the ShardReplicationAdd command.") } - keyspace, shard, err := topo.ParseKeyspaceShardString(subFlags.Arg(0)) + keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(0)) if err != nil { return err } @@ -1308,7 +1308,7 @@ func commandShardReplicationRemove(ctx context.Context, wr *wrangler.Wrangler, s return fmt.Errorf("The and arguments are required for the ShardReplicationRemove command.") } - keyspace, shard, err := topo.ParseKeyspaceShardString(subFlags.Arg(0)) + keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(0)) if err != nil { return err } @@ -1328,7 +1328,7 @@ func commandShardReplicationFix(ctx context.Context, wr *wrangler.Wrangler, subF } cell := subFlags.Arg(0) - keyspace, shard, err := topo.ParseKeyspaceShardString(subFlags.Arg(1)) + keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(1)) if err != nil { return err } @@ -1352,7 +1352,7 @@ func commandWaitForFilteredReplication(ctx context.Context, wr *wrangler.Wrangle if subFlags.NArg() != 1 { return fmt.Errorf("The argument is required for the WaitForFilteredReplication command.") } - keyspace, shard, err := topo.ParseKeyspaceShardString(subFlags.Arg(0)) + keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(0)) if err != nil { return err } @@ -1442,7 +1442,7 @@ func commandRemoveShardCell(ctx context.Context, wr *wrangler.Wrangler, subFlags return fmt.Errorf("The and arguments are required for the RemoveShardCell command.") } - keyspace, shard, err := topo.ParseKeyspaceShardString(subFlags.Arg(0)) + keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(0)) if err != nil { return err } @@ -1663,7 +1663,7 @@ func commandMigrateServedTypes(ctx context.Context, wr *wrangler.Wrangler, subFl return fmt.Errorf("The and arguments are both required for the MigrateServedTypes command.") } - keyspace, shard, err := topo.ParseKeyspaceShardString(subFlags.Arg(0)) + keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(0)) if err != nil { return err } @@ -1692,7 +1692,7 @@ func commandMigrateServedFrom(ctx context.Context, wr *wrangler.Wrangler, subFla return fmt.Errorf("The and arguments are both required for the MigrateServedFrom command.") } - keyspace, shard, err := topo.ParseKeyspaceShardString(subFlags.Arg(0)) + keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(0)) if err != nil { return err } @@ -1878,7 +1878,7 @@ func commandValidateSchemaShard(ctx context.Context, wr *wrangler.Wrangler, subF return fmt.Errorf("The argument is required for the ValidateSchemaShard command.") } - keyspace, shard, err := topo.ParseKeyspaceShardString(subFlags.Arg(0)) + keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(0)) if err != nil { return err } @@ -1950,12 +1950,12 @@ func commandCopySchemaShard(ctx context.Context, wr *wrangler.Wrangler, subFlags if *excludeTables != "" { excludeTableArray = strings.Split(*excludeTables, ",") } - destKeyspace, destShard, err := topo.ParseKeyspaceShardString(subFlags.Arg(1)) + destKeyspace, destShard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(1)) if err != nil { return err } - sourceKeyspace, sourceShard, err := topo.ParseKeyspaceShardString(subFlags.Arg(0)) + sourceKeyspace, sourceShard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(0)) if err == nil { return wr.CopySchemaShardFromShard(ctx, tableArray, excludeTableArray, *includeViews, sourceKeyspace, sourceShard, destKeyspace, destShard) } @@ -1974,7 +1974,7 @@ func commandValidateVersionShard(ctx context.Context, wr *wrangler.Wrangler, sub return fmt.Errorf("The argument is requird for the ValidateVersionShard command.") } - keyspace, shard, err := topo.ParseKeyspaceShardString(subFlags.Arg(0)) + keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(0)) if err != nil { return err } @@ -2019,7 +2019,7 @@ func commandValidatePermissionsShard(ctx context.Context, wr *wrangler.Wrangler, return fmt.Errorf("The argument is required for the ValidatePermissionsShard command.") } - keyspace, shard, err := topo.ParseKeyspaceShardString(subFlags.Arg(0)) + keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(0)) if err != nil { return err } @@ -2114,7 +2114,7 @@ func commandGetSrvShard(ctx context.Context, wr *wrangler.Wrangler, subFlags *fl return fmt.Errorf("The and arguments are required for the GetSrvShard command.") } - keyspace, shard, err := topo.ParseKeyspaceShardString(subFlags.Arg(1)) + keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(1)) if err != nil { return err } @@ -2133,7 +2133,7 @@ func commandGetEndPoints(ctx context.Context, wr *wrangler.Wrangler, subFlags *f return fmt.Errorf("The , , and arguments are required for the GetEndPoints command.") } - keyspace, shard, err := topo.ParseKeyspaceShardString(subFlags.Arg(1)) + keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(1)) if err != nil { return err } @@ -2156,7 +2156,7 @@ func commandGetShardReplication(ctx context.Context, wr *wrangler.Wrangler, subF return fmt.Errorf("The and arguments are required for the GetShardReplication command.") } - keyspace, shard, err := topo.ParseKeyspaceShardString(subFlags.Arg(1)) + keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(1)) if err != nil { return err } diff --git a/go/vt/worker/split_clone_cmd.go b/go/vt/worker/split_clone_cmd.go index 78d4911829..fc8c3c2814 100644 --- a/go/vt/worker/split_clone_cmd.go +++ b/go/vt/worker/split_clone_cmd.go @@ -14,7 +14,7 @@ import ( "sync" "github.com/youtube/vitess/go/vt/concurrency" - "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "github.com/youtube/vitess/go/vt/topotools" "github.com/youtube/vitess/go/vt/wrangler" "golang.org/x/net/context" @@ -95,7 +95,7 @@ func commandSplitClone(wi *Instance, wr *wrangler.Wrangler, subFlags *flag.FlagS return nil, fmt.Errorf("command SplitClone requires ") } - keyspace, shard, err := topo.ParseKeyspaceShardString(subFlags.Arg(0)) + keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(0)) if err != nil { return nil, err } diff --git a/go/vt/worker/split_diff_cmd.go b/go/vt/worker/split_diff_cmd.go index c3729bdf13..eccf4a9f6a 100644 --- a/go/vt/worker/split_diff_cmd.go +++ b/go/vt/worker/split_diff_cmd.go @@ -13,7 +13,7 @@ import ( "sync" "github.com/youtube/vitess/go/vt/concurrency" - "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "github.com/youtube/vitess/go/vt/wrangler" "golang.org/x/net/context" ) @@ -66,7 +66,7 @@ func commandSplitDiff(wi *Instance, wr *wrangler.Wrangler, subFlags *flag.FlagSe subFlags.Usage() return nil, fmt.Errorf("command SplitDiff requires ") } - keyspace, shard, err := topo.ParseKeyspaceShardString(subFlags.Arg(0)) + keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(0)) if err != nil { return nil, err } diff --git a/go/vt/worker/vertical_split_clone_cmd.go b/go/vt/worker/vertical_split_clone_cmd.go index 2c6247afd2..ba611d9c6f 100644 --- a/go/vt/worker/vertical_split_clone_cmd.go +++ b/go/vt/worker/vertical_split_clone_cmd.go @@ -14,7 +14,7 @@ import ( "sync" "github.com/youtube/vitess/go/vt/concurrency" - "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "github.com/youtube/vitess/go/vt/wrangler" "golang.org/x/net/context" ) @@ -93,7 +93,7 @@ func commandVerticalSplitClone(wi *Instance, wr *wrangler.Wrangler, subFlags *fl return nil, fmt.Errorf("command VerticalSplitClone requires ") } - keyspace, shard, err := topo.ParseKeyspaceShardString(subFlags.Arg(0)) + keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(0)) if err != nil { return nil, err } diff --git a/go/vt/worker/vertical_split_diff_cmd.go b/go/vt/worker/vertical_split_diff_cmd.go index 7c06b99c38..9ec120ef49 100644 --- a/go/vt/worker/vertical_split_diff_cmd.go +++ b/go/vt/worker/vertical_split_diff_cmd.go @@ -13,7 +13,7 @@ import ( "sync" "github.com/youtube/vitess/go/vt/concurrency" - "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/topo/topoproto" "github.com/youtube/vitess/go/vt/wrangler" "golang.org/x/net/context" ) @@ -65,7 +65,7 @@ func commandVerticalSplitDiff(wi *Instance, wr *wrangler.Wrangler, subFlags *fla subFlags.Usage() return nil, fmt.Errorf("command VerticalSplitDiff requires ") } - keyspace, shard, err := topo.ParseKeyspaceShardString(subFlags.Arg(0)) + keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(0)) if err != nil { return nil, err } From 566ba64c09185233111188a7272566b95e17bc2e Mon Sep 17 00:00:00 2001 From: Alain Jobart Date: Thu, 20 Aug 2015 09:58:49 -0700 Subject: [PATCH 76/80] Now fixing UpdateTablet. --- go/vt/etcdtopo/tablet.go | 20 +++++--------------- go/vt/topo/helpers/tee.go | 4 ++-- go/vt/topo/server.go | 2 +- go/vt/topo/tablet.go | 17 +++++++++++++---- go/vt/topo/test/faketopo/faketopo.go | 2 +- go/vt/zktopo/tablet.go | 9 ++------- 6 files changed, 24 insertions(+), 30 deletions(-) diff --git a/go/vt/etcdtopo/tablet.go b/go/vt/etcdtopo/tablet.go index ead6967730..dba9101953 100644 --- a/go/vt/etcdtopo/tablet.go +++ b/go/vt/etcdtopo/tablet.go @@ -32,26 +32,21 @@ func (s *Server) CreateTablet(ctx context.Context, tablet *pb.Tablet) error { if err != nil { return convertError(err) } - - event.Dispatch(&events.TabletChange{ - Tablet: *tablet, - Status: "created", - }) return nil } // UpdateTablet implements topo.Server. -func (s *Server) UpdateTablet(ctx context.Context, ti *topo.TabletInfo, existingVersion int64) (int64, error) { - cell, err := s.getCell(ti.Alias.Cell) +func (s *Server) UpdateTablet(ctx context.Context, tablet *pb.Tablet, existingVersion int64) (int64, error) { + cell, err := s.getCell(tablet.Alias.Cell) if err != nil { return -1, err } - data, err := json.MarshalIndent(ti.Tablet, "", " ") + data, err := json.MarshalIndent(tablet, "", " ") if err != nil { return -1, err } - resp, err := cell.CompareAndSwap(tabletFilePath(ti.Alias), + resp, err := cell.CompareAndSwap(tabletFilePath(tablet.Alias), string(data), 0 /* ttl */, "" /* prevValue */, uint64(existingVersion)) if err != nil { return -1, convertError(err) @@ -59,11 +54,6 @@ func (s *Server) UpdateTablet(ctx context.Context, ti *topo.TabletInfo, existing if resp.Node == nil { return -1, ErrBadResponse } - - event.Dispatch(&events.TabletChange{ - Tablet: *ti.Tablet, - Status: "updated", - }) return int64(resp.Node.ModifiedIndex), nil } @@ -79,7 +69,7 @@ func (s *Server) UpdateTabletFields(ctx context.Context, tabletAlias *pb.TabletA if err = updateFunc(ti.Tablet); err != nil { return err } - if _, err = s.UpdateTablet(ctx, ti, ti.Version()); err != topo.ErrBadVersion { + if _, err = s.UpdateTablet(ctx, ti.Tablet, ti.Version()); err != topo.ErrBadVersion { break } } diff --git a/go/vt/topo/helpers/tee.go b/go/vt/topo/helpers/tee.go index 3c9f5484cf..f6eab55e40 100644 --- a/go/vt/topo/helpers/tee.go +++ b/go/vt/topo/helpers/tee.go @@ -366,7 +366,7 @@ func (tee *Tee) CreateTablet(ctx context.Context, tablet *pb.Tablet) error { } // UpdateTablet is part of the topo.Server interface -func (tee *Tee) UpdateTablet(ctx context.Context, tablet *topo.TabletInfo, existingVersion int64) (newVersion int64, err error) { +func (tee *Tee) UpdateTablet(ctx context.Context, tablet *pb.Tablet, existingVersion int64) (newVersion int64, err error) { if newVersion, err = tee.primary.UpdateTablet(ctx, tablet, existingVersion); err != nil { // failed on primary, not updating secondary return @@ -387,7 +387,7 @@ func (tee *Tee) UpdateTablet(ctx context.Context, tablet *topo.TabletInfo, exist if serr == topo.ErrNoNode { // the tablet doesn't exist on the secondary, let's // just create it - if serr = tee.secondary.CreateTablet(ctx, tablet.Tablet); serr != nil { + if serr = tee.secondary.CreateTablet(ctx, tablet); serr != nil { log.Warningf("secondary.CreateTablet(%v) failed (after UpdateTablet returned ErrNoNode): %v", tablet.Alias, serr) } else { log.Infof("secondary.UpdateTablet(%v) failed with ErrNoNode, CreateTablet then worked.", tablet.Alias) diff --git a/go/vt/topo/server.go b/go/vt/topo/server.go index fce3bb9e6e..7dada7cf15 100644 --- a/go/vt/topo/server.go +++ b/go/vt/topo/server.go @@ -148,7 +148,7 @@ type Impl interface { // has changed. // // Do not use directly, but instead use topo.UpdateTablet. - UpdateTablet(ctx context.Context, tablet *TabletInfo, existingVersion int64) (newVersion int64, err error) + UpdateTablet(ctx context.Context, tablet *pb.Tablet, existingVersion int64) (newVersion int64, err error) // UpdateTabletFields updates the current tablet record // with new values, independently of the version diff --git a/go/vt/topo/tablet.go b/go/vt/topo/tablet.go index 8e2483a84a..7671344708 100644 --- a/go/vt/topo/tablet.go +++ b/go/vt/topo/tablet.go @@ -12,10 +12,12 @@ import ( "golang.org/x/net/context" log "github.com/golang/glog" + "github.com/youtube/vitess/go/event" "github.com/youtube/vitess/go/netutil" "github.com/youtube/vitess/go/trace" pb "github.com/youtube/vitess/go/vt/proto/topodata" + "github.com/youtube/vitess/go/vt/topo/events" "github.com/youtube/vitess/go/vt/topo/topoproto" ) @@ -296,6 +298,7 @@ func GetTablet(ctx context.Context, ts Server, alias *pb.TabletAlias) (*TabletIn } // UpdateTablet updates the tablet data only - not associated replication paths. +// It also uses a span, and sends the event. func UpdateTablet(ctx context.Context, ts Server, tablet *TabletInfo) error { span := trace.NewSpanFromContext(ctx) span.StartClient("TopoServer.UpdateTablet") @@ -307,11 +310,17 @@ func UpdateTablet(ctx context.Context, ts Server, tablet *TabletInfo) error { version = tablet.version } - newVersion, err := ts.UpdateTablet(ctx, tablet, version) - if err == nil { - tablet.version = newVersion + newVersion, err := ts.UpdateTablet(ctx, tablet.Tablet, version) + if err != nil { + return err } - return err + tablet.version = newVersion + + event.Dispatch(&events.TabletChange{ + Tablet: *tablet.Tablet, + Status: "updated", + }) + return nil } // UpdateTabletFields is a high level wrapper for TopoServer.UpdateTabletFields diff --git a/go/vt/topo/test/faketopo/faketopo.go b/go/vt/topo/test/faketopo/faketopo.go index a7280444ac..583a1b5193 100644 --- a/go/vt/topo/test/faketopo/faketopo.go +++ b/go/vt/topo/test/faketopo/faketopo.go @@ -105,7 +105,7 @@ func (ft FakeTopo) CreateTablet(ctx context.Context, tablet *pb.Tablet) error { } // UpdateTablet implements topo.Server. -func (ft FakeTopo) UpdateTablet(ctx context.Context, tablet *topo.TabletInfo, existingVersion int64) (newVersion int64, err error) { +func (ft FakeTopo) UpdateTablet(ctx context.Context, tablet *pb.Tablet, existingVersion int64) (newVersion int64, err error) { return 0, errNotImplemented } diff --git a/go/vt/zktopo/tablet.go b/go/vt/zktopo/tablet.go index 50c1199cd0..6876378840 100644 --- a/go/vt/zktopo/tablet.go +++ b/go/vt/zktopo/tablet.go @@ -76,9 +76,9 @@ func (zkts *Server) CreateTablet(ctx context.Context, tablet *pb.Tablet) error { } // UpdateTablet is part of the topo.Server interface -func (zkts *Server) UpdateTablet(ctx context.Context, tablet *topo.TabletInfo, existingVersion int64) (int64, error) { +func (zkts *Server) UpdateTablet(ctx context.Context, tablet *pb.Tablet, existingVersion int64) (int64, error) { zkTabletPath := TabletPathForAlias(tablet.Alias) - data, err := json.MarshalIndent(tablet.Tablet, " ", " ") + data, err := json.MarshalIndent(tablet, " ", " ") if err != nil { return 0, err } @@ -93,11 +93,6 @@ func (zkts *Server) UpdateTablet(ctx context.Context, tablet *topo.TabletInfo, e return 0, err } - - event.Dispatch(&events.TabletChange{ - Tablet: *tablet.Tablet, - Status: "updated", - }) return int64(stat.Version()), nil } From 86d08dbed6837695f46344133d6161667b485c67 Mon Sep 17 00:00:00 2001 From: Alain Jobart Date: Thu, 20 Aug 2015 10:25:44 -0700 Subject: [PATCH 77/80] Fixing topo.GetTablet. --- go/vt/etcdtopo/tablet.go | 35 ++++++++++---------- go/vt/schemamanager/schemamanager_test.go | 12 +++---- go/vt/tabletmanager/agent.go | 2 +- go/vt/topo/helpers/copy.go | 8 ++--- go/vt/topo/helpers/tee.go | 20 ++++++------ go/vt/topo/server.go | 2 +- go/vt/topo/tablet.go | 11 +++++-- go/vt/topo/test/faketopo/faketopo.go | 4 +-- go/vt/topo/test/tablet.go | 26 +++++++-------- go/vt/worker/clone_utils.go | 2 +- go/vt/zktopo/tablet.go | 40 ++++++++--------------- 11 files changed, 78 insertions(+), 84 deletions(-) diff --git a/go/vt/etcdtopo/tablet.go b/go/vt/etcdtopo/tablet.go index dba9101953..5d733feb88 100644 --- a/go/vt/etcdtopo/tablet.go +++ b/go/vt/etcdtopo/tablet.go @@ -59,17 +59,18 @@ func (s *Server) UpdateTablet(ctx context.Context, tablet *pb.Tablet, existingVe // UpdateTabletFields implements topo.Server. func (s *Server) UpdateTabletFields(ctx context.Context, tabletAlias *pb.TabletAlias, updateFunc func(*pb.Tablet) error) error { - var ti *topo.TabletInfo + var tablet *pb.Tablet var err error for { - if ti, err = s.GetTablet(ctx, tabletAlias); err != nil { + var version int64 + if tablet, version, err = s.GetTablet(ctx, tabletAlias); err != nil { return err } - if err = updateFunc(ti.Tablet); err != nil { + if err = updateFunc(tablet); err != nil { return err } - if _, err = s.UpdateTablet(ctx, ti.Tablet, ti.Version()); err != topo.ErrBadVersion { + if _, err = s.UpdateTablet(ctx, tablet, version); err != topo.ErrBadVersion { break } } @@ -78,7 +79,7 @@ func (s *Server) UpdateTabletFields(ctx context.Context, tabletAlias *pb.TabletA } event.Dispatch(&events.TabletChange{ - Tablet: *ti.Tablet, + Tablet: *tablet, Status: "updated", }) return nil @@ -92,7 +93,7 @@ func (s *Server) DeleteTablet(ctx context.Context, tabletAlias *pb.TabletAlias) } // Get the keyspace and shard names for the TabletChange event. - ti, tiErr := s.GetTablet(ctx, tabletAlias) + tablet, _, tErr := s.GetTablet(ctx, tabletAlias) _, err = cell.Delete(tabletDirPath(tabletAlias), true /* recursive */) if err != nil { @@ -100,13 +101,13 @@ func (s *Server) DeleteTablet(ctx context.Context, tabletAlias *pb.TabletAlias) } // Only try to log if we have the required info. - if tiErr == nil { + if tErr == nil { // Only copy the identity info for the tablet. The rest has been deleted. event.Dispatch(&events.TabletChange{ Tablet: pb.Tablet{ - Alias: ti.Tablet.Alias, - Keyspace: ti.Tablet.Keyspace, - Shard: ti.Tablet.Shard, + Alias: tablet.Alias, + Keyspace: tablet.Keyspace, + Shard: tablet.Shard, }, Status: "deleted", }) @@ -116,31 +117,31 @@ func (s *Server) DeleteTablet(ctx context.Context, tabletAlias *pb.TabletAlias) // ValidateTablet implements topo.Server. func (s *Server) ValidateTablet(ctx context.Context, tabletAlias *pb.TabletAlias) error { - _, err := s.GetTablet(ctx, tabletAlias) + _, _, err := s.GetTablet(ctx, tabletAlias) return err } // GetTablet implements topo.Server. -func (s *Server) GetTablet(ctx context.Context, tabletAlias *pb.TabletAlias) (*topo.TabletInfo, error) { +func (s *Server) GetTablet(ctx context.Context, tabletAlias *pb.TabletAlias) (*pb.Tablet, int64, error) { cell, err := s.getCell(tabletAlias.Cell) if err != nil { - return nil, err + return nil, 0, err } resp, err := cell.Get(tabletFilePath(tabletAlias), false /* sort */, false /* recursive */) if err != nil { - return nil, convertError(err) + return nil, 0, convertError(err) } if resp.Node == nil { - return nil, ErrBadResponse + return nil, 0, ErrBadResponse } value := &pb.Tablet{} if err := json.Unmarshal([]byte(resp.Node.Value), value); err != nil { - return nil, fmt.Errorf("bad tablet data (%v): %q", err, resp.Node.Value) + return nil, 0, fmt.Errorf("bad tablet data (%v): %q", err, resp.Node.Value) } - return topo.NewTabletInfo(value, int64(resp.Node.ModifiedIndex)), nil + return value, int64(resp.Node.ModifiedIndex), nil } // GetTabletsByCell implements topo.Server. diff --git a/go/vt/schemamanager/schemamanager_test.go b/go/vt/schemamanager/schemamanager_test.go index 183b1ded1f..0b6f5fee19 100644 --- a/go/vt/schemamanager/schemamanager_test.go +++ b/go/vt/schemamanager/schemamanager_test.go @@ -300,13 +300,11 @@ func (topoServer *fakeTopo) GetShard(ctx context.Context, keyspace string, shard return value, 0, nil } -func (topoServer *fakeTopo) GetTablet(ctx context.Context, tabletAlias *pb.TabletAlias) (*topo.TabletInfo, error) { - return &topo.TabletInfo{ - Tablet: &pb.Tablet{ - Alias: tabletAlias, - Keyspace: "test_keyspace", - }, - }, nil +func (topoServer *fakeTopo) GetTablet(ctx context.Context, tabletAlias *pb.TabletAlias) (*pb.Tablet, int64, error) { + return &pb.Tablet{ + Alias: tabletAlias, + Keyspace: "test_keyspace", + }, 0, nil } type fakeController struct { diff --git a/go/vt/tabletmanager/agent.go b/go/vt/tabletmanager/agent.go index 26ec7f13e1..011905ba76 100644 --- a/go/vt/tabletmanager/agent.go +++ b/go/vt/tabletmanager/agent.go @@ -259,7 +259,7 @@ func (agent *ActionAgent) updateState(ctx context.Context, oldTablet *pb.Tablet, } func (agent *ActionAgent) readTablet(ctx context.Context) (*topo.TabletInfo, error) { - tablet, err := topo.GetTablet(ctx, agent.TopoServer, agent.TabletAlias) + tablet, err := agent.TopoServer.GetTablet(ctx, agent.TabletAlias) if err != nil { return nil, err } diff --git a/go/vt/topo/helpers/copy.go b/go/vt/topo/helpers/copy.go index 8824be72b7..d982c96988 100644 --- a/go/vt/topo/helpers/copy.go +++ b/go/vt/topo/helpers/copy.go @@ -140,19 +140,19 @@ func CopyTablets(ctx context.Context, fromTS, toTS topo.Impl) { defer wg.Done() // read the source tablet - ti, err := fromTS.GetTablet(ctx, tabletAlias) + tablet, _, err := fromTS.GetTablet(ctx, tabletAlias) if err != nil { rec.RecordError(fmt.Errorf("GetTablet(%v): %v", tabletAlias, err)) return } // try to create the destination - err = toTS.CreateTablet(ctx, ti.Tablet) + err = toTS.CreateTablet(ctx, tablet) if err == topo.ErrNodeExists { // update the destination tablet log.Warningf("tablet %v already exists, updating it", tabletAlias) - err = toTS.UpdateTabletFields(ctx, ti.Alias, func(t *pb.Tablet) error { - *t = *ti.Tablet + err = toTS.UpdateTabletFields(ctx, tablet.Alias, func(t *pb.Tablet) error { + *t = *tablet return nil }) } diff --git a/go/vt/topo/helpers/tee.go b/go/vt/topo/helpers/tee.go index f6eab55e40..416f46504c 100644 --- a/go/vt/topo/helpers/tee.go +++ b/go/vt/topo/helpers/tee.go @@ -391,14 +391,14 @@ func (tee *Tee) UpdateTablet(ctx context.Context, tablet *pb.Tablet, existingVer log.Warningf("secondary.CreateTablet(%v) failed (after UpdateTablet returned ErrNoNode): %v", tablet.Alias, serr) } else { log.Infof("secondary.UpdateTablet(%v) failed with ErrNoNode, CreateTablet then worked.", tablet.Alias) - ti, gerr := tee.secondary.GetTablet(ctx, tablet.Alias) + _, v, gerr := tee.secondary.GetTablet(ctx, tablet.Alias) if gerr != nil { log.Warningf("Failed to re-read tablet(%v) after creating it on secondary: %v", tablet.Alias, gerr) } else { tee.mu.Lock() tee.tabletVersionMapping[*tablet.Alias] = versionMapping{ readFromVersion: newVersion, - readFromSecondVersion: ti.Version(), + readFromSecondVersion: v, } tee.mu.Unlock() } @@ -445,25 +445,25 @@ func (tee *Tee) DeleteTablet(ctx context.Context, alias *pb.TabletAlias) error { } // GetTablet is part of the topo.Server interface -func (tee *Tee) GetTablet(ctx context.Context, alias *pb.TabletAlias) (*topo.TabletInfo, error) { - ti, err := tee.readFrom.GetTablet(ctx, alias) +func (tee *Tee) GetTablet(ctx context.Context, alias *pb.TabletAlias) (*pb.Tablet, int64, error) { + t, v, err := tee.readFrom.GetTablet(ctx, alias) if err != nil { - return nil, err + return nil, 0, err } - ti2, err := tee.readFromSecond.GetTablet(ctx, alias) + _, v2, err := tee.readFromSecond.GetTablet(ctx, alias) if err != nil { // can't read from secondary, so we can's keep version map - return ti, nil + return t, v, nil } tee.mu.Lock() tee.tabletVersionMapping[*alias] = versionMapping{ - readFromVersion: ti.Version(), - readFromSecondVersion: ti2.Version(), + readFromVersion: v, + readFromSecondVersion: v2, } tee.mu.Unlock() - return ti, nil + return t, v, nil } // GetTabletsByCell is part of the topo.Server interface diff --git a/go/vt/topo/server.go b/go/vt/topo/server.go index 7dada7cf15..2a4317adf9 100644 --- a/go/vt/topo/server.go +++ b/go/vt/topo/server.go @@ -163,7 +163,7 @@ type Impl interface { // GetTablet returns the tablet data (includes the current version). // Can return ErrNoNode if the tablet doesn't exist. - GetTablet(ctx context.Context, alias *pb.TabletAlias) (*TabletInfo, error) + GetTablet(ctx context.Context, alias *pb.TabletAlias) (*pb.Tablet, int64, error) // GetTabletsByCell returns all the tablets in the given cell. // Can return ErrNoNode if no tablet was ever created in that cell. diff --git a/go/vt/topo/tablet.go b/go/vt/topo/tablet.go index 7671344708..d384dc0da7 100644 --- a/go/vt/topo/tablet.go +++ b/go/vt/topo/tablet.go @@ -288,13 +288,20 @@ func NewTabletInfo(tablet *pb.Tablet, version int64) *TabletInfo { // GetTablet is a high level function to read tablet data. // It generates trace spans. -func GetTablet(ctx context.Context, ts Server, alias *pb.TabletAlias) (*TabletInfo, error) { +func (ts Server) GetTablet(ctx context.Context, alias *pb.TabletAlias) (*TabletInfo, error) { span := trace.NewSpanFromContext(ctx) span.StartClient("TopoServer.GetTablet") span.Annotate("tablet", topoproto.TabletAliasString(alias)) defer span.Finish() - return ts.GetTablet(ctx, alias) + value, version, err := ts.Impl.GetTablet(ctx, alias) + if err != nil { + return nil, err + } + return &TabletInfo{ + version: version, + Tablet: value, + }, nil } // UpdateTablet updates the tablet data only - not associated replication paths. diff --git a/go/vt/topo/test/faketopo/faketopo.go b/go/vt/topo/test/faketopo/faketopo.go index 583a1b5193..2c7ea25f29 100644 --- a/go/vt/topo/test/faketopo/faketopo.go +++ b/go/vt/topo/test/faketopo/faketopo.go @@ -120,8 +120,8 @@ func (ft FakeTopo) DeleteTablet(ctx context.Context, alias *pb.TabletAlias) erro } // GetTablet implements topo.Server. -func (ft FakeTopo) GetTablet(ctx context.Context, alias *pb.TabletAlias) (*topo.TabletInfo, error) { - return nil, errNotImplemented +func (ft FakeTopo) GetTablet(ctx context.Context, alias *pb.TabletAlias) (*pb.Tablet, int64, error) { + return nil, 0, errNotImplemented } // GetTabletsByCell implements topo.Server. diff --git a/go/vt/topo/test/tablet.go b/go/vt/topo/test/tablet.go index a36a072d5b..a475bb72c8 100644 --- a/go/vt/topo/test/tablet.go +++ b/go/vt/topo/test/tablet.go @@ -51,18 +51,18 @@ func CheckTablet(ctx context.Context, t *testing.T, ts topo.Impl) { t.Errorf("CreateTablet(again): %v", err) } - if _, err := ts.GetTablet(ctx, &pb.TabletAlias{Cell: cell, Uid: 666}); err != topo.ErrNoNode { + if _, _, err := ts.GetTablet(ctx, &pb.TabletAlias{Cell: cell, Uid: 666}); err != topo.ErrNoNode { t.Errorf("GetTablet(666): %v", err) } - ti, err := ts.GetTablet(ctx, tablet.Alias) + nt, nv, err := ts.GetTablet(ctx, tablet.Alias) if err != nil { t.Errorf("GetTablet %v: %v", tablet.Alias, err) } - if eq, err := tabletEqual(ti.Tablet, tablet); err != nil { + if eq, err := tabletEqual(nt, tablet); err != nil { t.Errorf("cannot compare tablets: %v", err) } else if !eq { - t.Errorf("put and got tablets are not identical:\n%#v\n%#v", tablet, ti.Tablet) + t.Errorf("put and got tablets are not identical:\n%#v\n%#v", tablet, t) } if _, err := ts.GetTabletsByCell(ctx, "666"); err != topo.ErrNoNode { @@ -77,17 +77,17 @@ func CheckTablet(ctx context.Context, t *testing.T, ts topo.Impl) { t.Errorf("GetTabletsByCell: want [%v], got %v", tablet.Alias, inCell) } - ti.Hostname = "remotehost" - if err := topo.UpdateTablet(ctx, topo.Server{Impl: ts}, ti); err != nil { + nt.Hostname = "remotehost" + if _, err := ts.UpdateTablet(ctx, nt, nv); err != nil { t.Errorf("UpdateTablet: %v", err) } - ti, err = ts.GetTablet(ctx, tablet.Alias) + nt, nv, err = ts.GetTablet(ctx, tablet.Alias) if err != nil { t.Errorf("GetTablet %v: %v", tablet.Alias, err) } - if want := "remotehost"; ti.Hostname != want { - t.Errorf("ti.Hostname: want %v, got %v", want, ti.Hostname) + if want := "remotehost"; nt.Hostname != want { + t.Errorf("nt.Hostname: want %v, got %v", want, nt.Hostname) } if err := topo.UpdateTabletFields(ctx, topo.Server{Impl: ts}, tablet.Alias, func(t *pb.Tablet) error { @@ -96,13 +96,13 @@ func CheckTablet(ctx context.Context, t *testing.T, ts topo.Impl) { }); err != nil { t.Errorf("UpdateTabletFields: %v", err) } - ti, err = ts.GetTablet(ctx, tablet.Alias) + nt, nv, err = ts.GetTablet(ctx, tablet.Alias) if err != nil { t.Errorf("GetTablet %v: %v", tablet.Alias, err) } - if want := "anotherhost"; ti.Hostname != want { - t.Errorf("ti.Hostname: want %v, got %v", want, ti.Hostname) + if want := "anotherhost"; nt.Hostname != want { + t.Errorf("nt.Hostname: want %v, got %v", want, nt.Hostname) } if err := ts.DeleteTablet(ctx, tablet.Alias); err != nil { @@ -112,7 +112,7 @@ func CheckTablet(ctx context.Context, t *testing.T, ts topo.Impl) { t.Errorf("DeleteTablet(again): %v", err) } - if _, err := ts.GetTablet(ctx, tablet.Alias); err != topo.ErrNoNode { + if _, _, err := ts.GetTablet(ctx, tablet.Alias); err != topo.ErrNoNode { t.Errorf("GetTablet: expected error, tablet was deleted: %v", err) } diff --git a/go/vt/worker/clone_utils.go b/go/vt/worker/clone_utils.go index 7c60e6f836..09eb33dfdf 100644 --- a/go/vt/worker/clone_utils.go +++ b/go/vt/worker/clone_utils.go @@ -47,7 +47,7 @@ func resolveDestinationShardMaster(ctx context.Context, keyspace, shard string, wr.Logger().Infof("Found target master alias %v in shard %v/%v", topoproto.TabletAliasString(si.MasterAlias), keyspace, shard) shortCtx, cancel = context.WithTimeout(ctx, *remoteActionsTimeout) - ti, err = topo.GetTablet(shortCtx, wr.TopoServer(), si.MasterAlias) + ti, err = wr.TopoServer().GetTablet(shortCtx, si.MasterAlias) cancel() if err != nil { return ti, fmt.Errorf("unable to get master tablet from alias %v in shard %v/%v", diff --git a/go/vt/zktopo/tablet.go b/go/vt/zktopo/tablet.go index 6876378840..fe5d298484 100644 --- a/go/vt/zktopo/tablet.go +++ b/go/vt/zktopo/tablet.go @@ -33,23 +33,6 @@ func tabletDirectoryForCell(cell string) string { return fmt.Sprintf("/zk/%v/vt/tablets", cell) } -func tabletFromJSON(data string) (*pb.Tablet, error) { - t := &pb.Tablet{} - err := json.Unmarshal([]byte(data), t) - if err != nil { - return nil, err - } - return t, nil -} - -func tabletInfoFromJSON(data string, version int64) (*topo.TabletInfo, error) { - tablet, err := tabletFromJSON(data) - if err != nil { - return nil, err - } - return topo.NewTabletInfo(tablet, version), nil -} - // CreateTablet is part of the topo.Server interface func (zkts *Server) CreateTablet(ctx context.Context, tablet *pb.Tablet) error { zkTabletPath := TabletPathForAlias(tablet.Alias) @@ -107,8 +90,8 @@ func (zkts *Server) UpdateTabletFields(ctx context.Context, tabletAlias *pb.Tabl return "", fmt.Errorf("no data for tablet addr update: %v", tabletAlias) } - tablet, err := tabletFromJSON(oldValue) - if err != nil { + tablet := &pb.Tablet{} + if err := json.Unmarshal([]byte(oldValue), tablet); err != nil { return "", err } if err := update(tablet); err != nil { @@ -142,7 +125,7 @@ func (zkts *Server) UpdateTabletFields(ctx context.Context, tabletAlias *pb.Tabl func (zkts *Server) DeleteTablet(ctx context.Context, alias *pb.TabletAlias) error { // We need to find out the keyspace and shard names because // those are required in the TabletChange event. - ti, tiErr := zkts.GetTablet(ctx, alias) + tablet, _, tiErr := zkts.GetTablet(ctx, alias) zkTabletPath := TabletPathForAlias(alias) err := zk.DeleteRecursive(zkts.zconn, zkTabletPath, -1) @@ -159,9 +142,9 @@ func (zkts *Server) DeleteTablet(ctx context.Context, alias *pb.TabletAlias) err // The rest has just been deleted, so it should be blank. event.Dispatch(&events.TabletChange{ Tablet: pb.Tablet{ - Alias: ti.Tablet.Alias, - Keyspace: ti.Tablet.Keyspace, - Shard: ti.Tablet.Shard, + Alias: tablet.Alias, + Keyspace: tablet.Keyspace, + Shard: tablet.Shard, }, Status: "deleted", }) @@ -170,16 +153,21 @@ func (zkts *Server) DeleteTablet(ctx context.Context, alias *pb.TabletAlias) err } // GetTablet is part of the topo.Server interface -func (zkts *Server) GetTablet(ctx context.Context, alias *pb.TabletAlias) (*topo.TabletInfo, error) { +func (zkts *Server) GetTablet(ctx context.Context, alias *pb.TabletAlias) (*pb.Tablet, int64, error) { zkTabletPath := TabletPathForAlias(alias) data, stat, err := zkts.zconn.Get(zkTabletPath) if err != nil { if zookeeper.IsError(err, zookeeper.ZNONODE) { err = topo.ErrNoNode } - return nil, err + return nil, 0, err } - return tabletInfoFromJSON(data, int64(stat.Version())) + + tablet := &pb.Tablet{} + if err := json.Unmarshal([]byte(data), tablet); err != nil { + return nil, 0, err + } + return tablet, int64(stat.Version()), nil } // GetTabletsByCell is part of the topo.Server interface From 7526ad684e118882b6696a4ddb604d1fec9c9ef4 Mon Sep 17 00:00:00 2001 From: Alain Jobart Date: Thu, 20 Aug 2015 10:47:07 -0700 Subject: [PATCH 78/80] Fixing topo.UpdateTablet and topo.UpdateTabletFields. --- go/cmd/vtctld/api_test.go | 5 ++--- go/vt/etcdtopo/tablet.go | 15 +++++---------- go/vt/tabletmanager/agent.go | 2 +- go/vt/tabletmanager/agent_rpc_actions.go | 8 ++++---- go/vt/tabletmanager/healthcheck_test.go | 3 +-- go/vt/tabletmanager/init_tablet.go | 4 ++-- go/vt/tabletmanager/reparent.go | 4 ++-- go/vt/topo/helpers/copy.go | 2 +- go/vt/topo/helpers/copy_test.go | 5 +++-- go/vt/topo/helpers/tee.go | 11 ++++++----- go/vt/topo/server.go | 2 +- go/vt/topo/tablet.go | 22 ++++++++++++++++------ go/vt/topo/test/faketopo/faketopo.go | 4 ++-- go/vt/topo/test/tablet.go | 2 +- go/vt/topotools/rebuild_test.go | 6 +++--- go/vt/topotools/reparent.go | 2 +- go/vt/topotools/tablet.go | 4 ++-- go/vt/wrangler/tablet.go | 6 +++--- go/vt/zktopo/tablet.go | 13 +++---------- 19 files changed, 59 insertions(+), 61 deletions(-) diff --git a/go/cmd/vtctld/api_test.go b/go/cmd/vtctld/api_test.go index a654aaeda9..474b58add6 100644 --- a/go/cmd/vtctld/api_test.go +++ b/go/cmd/vtctld/api_test.go @@ -16,7 +16,6 @@ import ( "golang.org/x/net/context" "github.com/youtube/vitess/go/vt/logutil" - "github.com/youtube/vitess/go/vt/topo" "github.com/youtube/vitess/go/vt/topotools" "github.com/youtube/vitess/go/vt/wrangler" "github.com/youtube/vitess/go/vt/zktopo" @@ -51,7 +50,7 @@ func TestAPI(t *testing.T) { KeyRange: &pb.KeyRange{Start: []byte{0x80}, End: nil}, }) - topo.CreateTablet(ctx, ts, &pb.Tablet{ + ts.CreateTablet(ctx, &pb.Tablet{ Alias: &pb.TabletAlias{Cell: "cell1", Uid: 100}, Keyspace: "ks1", Shard: "-80", @@ -59,7 +58,7 @@ func TestAPI(t *testing.T) { KeyRange: &pb.KeyRange{Start: nil, End: []byte{0x80}}, PortMap: map[string]int32{"vt": 100}, }) - topo.CreateTablet(ctx, ts, &pb.Tablet{ + ts.CreateTablet(ctx, &pb.Tablet{ Alias: &pb.TabletAlias{Cell: "cell2", Uid: 200}, Keyspace: "ks1", Shard: "-80", diff --git a/go/vt/etcdtopo/tablet.go b/go/vt/etcdtopo/tablet.go index 5d733feb88..d9cd7a9237 100644 --- a/go/vt/etcdtopo/tablet.go +++ b/go/vt/etcdtopo/tablet.go @@ -58,31 +58,26 @@ func (s *Server) UpdateTablet(ctx context.Context, tablet *pb.Tablet, existingVe } // UpdateTabletFields implements topo.Server. -func (s *Server) UpdateTabletFields(ctx context.Context, tabletAlias *pb.TabletAlias, updateFunc func(*pb.Tablet) error) error { +func (s *Server) UpdateTabletFields(ctx context.Context, tabletAlias *pb.TabletAlias, updateFunc func(*pb.Tablet) error) (*pb.Tablet, error) { var tablet *pb.Tablet var err error for { var version int64 if tablet, version, err = s.GetTablet(ctx, tabletAlias); err != nil { - return err + return nil, err } if err = updateFunc(tablet); err != nil { - return err + return nil, err } if _, err = s.UpdateTablet(ctx, tablet, version); err != topo.ErrBadVersion { break } } if err != nil { - return err + return nil, err } - - event.Dispatch(&events.TabletChange{ - Tablet: *tablet, - Status: "updated", - }) - return nil + return tablet, nil } // DeleteTablet implements topo.Server. diff --git a/go/vt/tabletmanager/agent.go b/go/vt/tabletmanager/agent.go index 011905ba76..d1f64b08cf 100644 --- a/go/vt/tabletmanager/agent.go +++ b/go/vt/tabletmanager/agent.go @@ -491,7 +491,7 @@ func (agent *ActionAgent) checkTabletMysqlPort(ctx context.Context, tablet *topo log.Warningf("MySQL port has changed from %v to %v, updating it in tablet record", tablet.PortMap["mysql"], mport) tablet.PortMap["mysql"] = mport - if err := topo.UpdateTablet(ctx, agent.TopoServer, tablet); err != nil { + if err := agent.TopoServer.UpdateTablet(ctx, tablet); err != nil { log.Warningf("Failed to update tablet record, may use old mysql port") return nil } diff --git a/go/vt/tabletmanager/agent_rpc_actions.go b/go/vt/tabletmanager/agent_rpc_actions.go index 5cdc7ecacf..d5b2bd8502 100644 --- a/go/vt/tabletmanager/agent_rpc_actions.go +++ b/go/vt/tabletmanager/agent_rpc_actions.go @@ -412,7 +412,7 @@ func (agent *ActionAgent) InitMaster(ctx context.Context) (myproto.ReplicationPo } // Change our type to master if not already - if err := topo.UpdateTabletFields(ctx, agent.TopoServer, agent.TabletAlias, func(tablet *pb.Tablet) error { + if err := agent.TopoServer.UpdateTabletFields(ctx, agent.TabletAlias, func(tablet *pb.Tablet) error { tablet.Type = pb.TabletType_MASTER tablet.HealthMap = nil return nil @@ -571,7 +571,7 @@ func (agent *ActionAgent) SetMaster(ctx context.Context, parent *pb.TabletAlias, if tablet.Type == pb.TabletType_MASTER { tablet.Type = pb.TabletType_SPARE tablet.HealthMap = nil - if err := topo.UpdateTablet(ctx, agent.TopoServer, tablet); err != nil { + if err := agent.TopoServer.UpdateTablet(ctx, tablet); err != nil { return err } } @@ -596,7 +596,7 @@ func (agent *ActionAgent) SlaveWasRestarted(ctx context.Context, swrd *actionnod if tablet.Type == pb.TabletType_MASTER { tablet.Type = pb.TabletType_SPARE } - err = topo.UpdateTablet(ctx, agent.TopoServer, tablet) + err = agent.TopoServer.UpdateTablet(ctx, tablet) if err != nil { return err } @@ -660,7 +660,7 @@ func (agent *ActionAgent) updateReplicationGraphForPromotedSlave(ctx context.Con // Update tablet regardless - trend towards consistency. tablet.Type = pb.TabletType_MASTER tablet.HealthMap = nil - err := topo.UpdateTablet(ctx, agent.TopoServer, tablet) + err := agent.TopoServer.UpdateTablet(ctx, tablet) if err != nil { return err } diff --git a/go/vt/tabletmanager/healthcheck_test.go b/go/vt/tabletmanager/healthcheck_test.go index ec171ae094..48bc19b793 100644 --- a/go/vt/tabletmanager/healthcheck_test.go +++ b/go/vt/tabletmanager/healthcheck_test.go @@ -11,7 +11,6 @@ import ( "github.com/youtube/vitess/go/vt/mysqlctl" "github.com/youtube/vitess/go/vt/tabletmanager/actionnode" "github.com/youtube/vitess/go/vt/tabletserver" - "github.com/youtube/vitess/go/vt/topo" "github.com/youtube/vitess/go/vt/zktopo" "golang.org/x/net/context" @@ -136,7 +135,7 @@ func createTestAgent(ctx context.Context, t *testing.T) *ActionAgent { Shard: shard, Type: pb.TabletType_SPARE, } - if err := topo.CreateTablet(ctx, ts, tablet); err != nil { + if err := ts.CreateTablet(ctx, tablet); err != nil { t.Fatalf("CreateTablet failed: %v", err) } diff --git a/go/vt/tabletmanager/init_tablet.go b/go/vt/tabletmanager/init_tablet.go index aa66a3ec92..0021d84d18 100644 --- a/go/vt/tabletmanager/init_tablet.go +++ b/go/vt/tabletmanager/init_tablet.go @@ -175,7 +175,7 @@ func (agent *ActionAgent) InitTablet(port, gRPCPort int32) error { } // now try to create the record - err := topo.CreateTablet(ctx, agent.TopoServer, tablet) + err := agent.TopoServer.CreateTablet(ctx, tablet) switch err { case nil: // it worked, we're good, can update the replication graph @@ -200,7 +200,7 @@ func (agent *ActionAgent) InitTablet(port, gRPCPort int32) error { // And overwrite the rest *(oldTablet.Tablet) = *tablet - if err := topo.UpdateTablet(ctx, agent.TopoServer, oldTablet); err != nil { + if err := agent.TopoServer.UpdateTablet(ctx, oldTablet); err != nil { return fmt.Errorf("UpdateTablet failed: %v", err) } diff --git a/go/vt/tabletmanager/reparent.go b/go/vt/tabletmanager/reparent.go index 329bf57546..f4b77f02d6 100644 --- a/go/vt/tabletmanager/reparent.go +++ b/go/vt/tabletmanager/reparent.go @@ -153,7 +153,7 @@ func (agent *ActionAgent) finalizeTabletExternallyReparented(ctx context.Context defer wg.Done() // Update our own record to master. var updatedTablet *pb.Tablet - err := topo.UpdateTabletFields(ctx, agent.TopoServer, agent.TabletAlias, + err := agent.TopoServer.UpdateTabletFields(ctx, agent.TabletAlias, func(tablet *pb.Tablet) error { tablet.Type = pb.TabletType_MASTER tablet.HealthMap = nil @@ -177,7 +177,7 @@ func (agent *ActionAgent) finalizeTabletExternallyReparented(ctx context.Context go func() { // Force the old master to spare. var oldMasterTablet *pb.Tablet - err := topo.UpdateTabletFields(ctx, agent.TopoServer, oldMasterAlias, + err := agent.TopoServer.UpdateTabletFields(ctx, oldMasterAlias, func(tablet *pb.Tablet) error { tablet.Type = pb.TabletType_SPARE oldMasterTablet = tablet diff --git a/go/vt/topo/helpers/copy.go b/go/vt/topo/helpers/copy.go index d982c96988..69dd664b0b 100644 --- a/go/vt/topo/helpers/copy.go +++ b/go/vt/topo/helpers/copy.go @@ -151,7 +151,7 @@ func CopyTablets(ctx context.Context, fromTS, toTS topo.Impl) { if err == topo.ErrNodeExists { // update the destination tablet log.Warningf("tablet %v already exists, updating it", tabletAlias) - err = toTS.UpdateTabletFields(ctx, tablet.Alias, func(t *pb.Tablet) error { + _, err = toTS.UpdateTabletFields(ctx, tablet.Alias, func(t *pb.Tablet) error { *t = *tablet return nil }) diff --git a/go/vt/topo/helpers/copy_test.go b/go/vt/topo/helpers/copy_test.go index 307da3fe59..efefbe62c8 100644 --- a/go/vt/topo/helpers/copy_test.go +++ b/go/vt/topo/helpers/copy_test.go @@ -40,7 +40,8 @@ func createSetup(ctx context.Context, t *testing.T) (topo.Impl, topo.Impl) { if err := fromTS.CreateShard(ctx, "test_keyspace", "0", &pb.Shard{Cells: []string{"test_cell"}}); err != nil { t.Fatalf("cannot create shard: %v", err) } - if err := topo.CreateTablet(ctx, topo.Server{Impl: fromTS}, &pb.Tablet{ + tts := topo.Server{Impl: fromTS} + if err := tts.CreateTablet(ctx, &pb.Tablet{ Alias: &pb.TabletAlias{ Cell: "test_cell", Uid: 123, @@ -60,7 +61,7 @@ func createSetup(ctx context.Context, t *testing.T) (topo.Impl, topo.Impl) { }); err != nil { t.Fatalf("cannot create master tablet: %v", err) } - if err := topo.CreateTablet(ctx, topo.Server{Impl: fromTS}, &pb.Tablet{ + if err := tts.CreateTablet(ctx, &pb.Tablet{ Alias: &pb.TabletAlias{ Cell: "test_cell", Uid: 234, diff --git a/go/vt/topo/helpers/tee.go b/go/vt/topo/helpers/tee.go index 416f46504c..795218253c 100644 --- a/go/vt/topo/helpers/tee.go +++ b/go/vt/topo/helpers/tee.go @@ -418,17 +418,18 @@ func (tee *Tee) UpdateTablet(ctx context.Context, tablet *pb.Tablet, existingVer } // UpdateTabletFields is part of the topo.Server interface -func (tee *Tee) UpdateTabletFields(ctx context.Context, tabletAlias *pb.TabletAlias, update func(*pb.Tablet) error) error { - if err := tee.primary.UpdateTabletFields(ctx, tabletAlias, update); err != nil { +func (tee *Tee) UpdateTabletFields(ctx context.Context, tabletAlias *pb.TabletAlias, update func(*pb.Tablet) error) (*pb.Tablet, error) { + tablet, err := tee.primary.UpdateTabletFields(ctx, tabletAlias, update) + if err != nil { // failed on primary, not updating secondary - return err + return nil, err } - if err := tee.secondary.UpdateTabletFields(ctx, tabletAlias, update); err != nil { + if _, err := tee.secondary.UpdateTabletFields(ctx, tabletAlias, update); err != nil { // not critical enough to fail log.Warningf("secondary.UpdateTabletFields(%v) failed: %v", tabletAlias, err) } - return nil + return tablet, nil } // DeleteTablet is part of the topo.Server interface diff --git a/go/vt/topo/server.go b/go/vt/topo/server.go index 2a4317adf9..79e8c4c2e0 100644 --- a/go/vt/topo/server.go +++ b/go/vt/topo/server.go @@ -153,7 +153,7 @@ type Impl interface { // UpdateTabletFields updates the current tablet record // with new values, independently of the version // Can return ErrNoNode if the tablet doesn't exist. - UpdateTabletFields(ctx context.Context, tabletAlias *pb.TabletAlias, update func(*pb.Tablet) error) error + UpdateTabletFields(ctx context.Context, tabletAlias *pb.TabletAlias, update func(*pb.Tablet) error) (*pb.Tablet, error) // DeleteTablet removes a tablet from the system. // We assume no RPC is currently running to it. diff --git a/go/vt/topo/tablet.go b/go/vt/topo/tablet.go index d384dc0da7..ebb604fa14 100644 --- a/go/vt/topo/tablet.go +++ b/go/vt/topo/tablet.go @@ -306,7 +306,7 @@ func (ts Server) GetTablet(ctx context.Context, alias *pb.TabletAlias) (*TabletI // UpdateTablet updates the tablet data only - not associated replication paths. // It also uses a span, and sends the event. -func UpdateTablet(ctx context.Context, ts Server, tablet *TabletInfo) error { +func (ts Server) UpdateTablet(ctx context.Context, tablet *TabletInfo) error { span := trace.NewSpanFromContext(ctx) span.StartClient("TopoServer.UpdateTablet") span.Annotate("tablet", topoproto.TabletAliasString(tablet.Alias)) @@ -317,7 +317,7 @@ func UpdateTablet(ctx context.Context, ts Server, tablet *TabletInfo) error { version = tablet.version } - newVersion, err := ts.UpdateTablet(ctx, tablet.Tablet, version) + newVersion, err := ts.Impl.UpdateTablet(ctx, tablet.Tablet, version) if err != nil { return err } @@ -332,13 +332,23 @@ func UpdateTablet(ctx context.Context, ts Server, tablet *TabletInfo) error { // UpdateTabletFields is a high level wrapper for TopoServer.UpdateTabletFields // that generates trace spans. -func UpdateTabletFields(ctx context.Context, ts Server, alias *pb.TabletAlias, update func(*pb.Tablet) error) error { +func (ts Server) UpdateTabletFields(ctx context.Context, alias *pb.TabletAlias, update func(*pb.Tablet) error) error { span := trace.NewSpanFromContext(ctx) span.StartClient("TopoServer.UpdateTabletFields") span.Annotate("tablet", topoproto.TabletAliasString(alias)) defer span.Finish() - return ts.UpdateTabletFields(ctx, alias, update) + tablet, err := ts.Impl.UpdateTabletFields(ctx, alias, update) + if err != nil { + return err + } + if tablet != nil { + event.Dispatch(&events.TabletChange{ + Tablet: *tablet, + Status: "updated", + }) + } + return nil } // Validate makes sure a tablet is represented correctly in the topology server. @@ -396,9 +406,9 @@ func Validate(ctx context.Context, ts Server, tabletAlias *pb.TabletAlias) error // CreateTablet creates a new tablet and all associated paths for the // replication graph. -func CreateTablet(ctx context.Context, ts Server, tablet *pb.Tablet) error { +func (ts Server) CreateTablet(ctx context.Context, tablet *pb.Tablet) error { // Have the Server create the tablet - err := ts.CreateTablet(ctx, tablet) + err := ts.Impl.CreateTablet(ctx, tablet) if err != nil { return err } diff --git a/go/vt/topo/test/faketopo/faketopo.go b/go/vt/topo/test/faketopo/faketopo.go index 2c7ea25f29..c4868f48ea 100644 --- a/go/vt/topo/test/faketopo/faketopo.go +++ b/go/vt/topo/test/faketopo/faketopo.go @@ -110,8 +110,8 @@ func (ft FakeTopo) UpdateTablet(ctx context.Context, tablet *pb.Tablet, existing } // UpdateTabletFields implements topo.Server. -func (ft FakeTopo) UpdateTabletFields(ctx context.Context, tabletAlias *pb.TabletAlias, update func(*pb.Tablet) error) error { - return errNotImplemented +func (ft FakeTopo) UpdateTabletFields(ctx context.Context, tabletAlias *pb.TabletAlias, update func(*pb.Tablet) error) (*pb.Tablet, error) { + return nil, errNotImplemented } // DeleteTablet implements topo.Server. diff --git a/go/vt/topo/test/tablet.go b/go/vt/topo/test/tablet.go index a475bb72c8..9d5cc81537 100644 --- a/go/vt/topo/test/tablet.go +++ b/go/vt/topo/test/tablet.go @@ -90,7 +90,7 @@ func CheckTablet(ctx context.Context, t *testing.T, ts topo.Impl) { t.Errorf("nt.Hostname: want %v, got %v", want, nt.Hostname) } - if err := topo.UpdateTabletFields(ctx, topo.Server{Impl: ts}, tablet.Alias, func(t *pb.Tablet) error { + if _, err := ts.UpdateTabletFields(ctx, tablet.Alias, func(t *pb.Tablet) error { t.Hostname = "anotherhost" return nil }); err != nil { diff --git a/go/vt/topotools/rebuild_test.go b/go/vt/topotools/rebuild_test.go index 574dc0be1a..ddc444949e 100644 --- a/go/vt/topotools/rebuild_test.go +++ b/go/vt/topotools/rebuild_test.go @@ -39,7 +39,7 @@ func addTablet(ctx context.Context, t *testing.T, ts topo.Server, uid int, cell Type: tabletType, Shard: testShard, } - if err := topo.CreateTablet(ctx, ts, tablet); err != nil { + if err := ts.CreateTablet(ctx, tablet); err != nil { t.Fatalf("CreateTablet: %v", err) } @@ -92,7 +92,7 @@ func TestRebuildShard(t *testing.T) { // Make a change. masterInfo.Type = pb.TabletType_SPARE - if err := topo.UpdateTablet(ctx, ts, masterInfo); err != nil { + if err := ts.UpdateTablet(ctx, masterInfo); err != nil { t.Fatalf("UpdateTablet: %v", err) } if _, err := RebuildShard(ctx, logger, ts, testKeyspace, testShard, cells, time.Minute); err != nil { @@ -101,7 +101,7 @@ func TestRebuildShard(t *testing.T) { // Make another change. replicaInfo.Type = pb.TabletType_SPARE - if err := topo.UpdateTablet(ctx, ts, replicaInfo); err != nil { + if err := ts.UpdateTablet(ctx, replicaInfo); err != nil { t.Fatalf("UpdateTablet: %v", err) } if _, err := RebuildShard(ctx, logger, ts, testKeyspace, testShard, cells, time.Minute); err != nil { diff --git a/go/vt/topotools/reparent.go b/go/vt/topotools/reparent.go index bd7726001d..adf401f7aa 100644 --- a/go/vt/topotools/reparent.go +++ b/go/vt/topotools/reparent.go @@ -56,7 +56,7 @@ func RestartSlavesExternal(ts topo.Server, log logutil.Logger, slaveTabletMap, m log.Warningf("Old master %v is not restarting in time, forcing it to spare: %v", ti.Alias, err) ti.Type = pb.TabletType_SPARE - if err := topo.UpdateTablet(context.TODO(), ts, ti); err != nil { + if err := ts.UpdateTablet(context.TODO(), ti); err != nil { log.Warningf("Failed to change old master %v to spare: %v", ti.Alias, err) } } diff --git a/go/vt/topotools/tablet.go b/go/vt/topotools/tablet.go index b82b367d22..893d48eeeb 100644 --- a/go/vt/topotools/tablet.go +++ b/go/vt/topotools/tablet.go @@ -61,7 +61,7 @@ func Scrap(ctx context.Context, ts topo.Server, tabletAlias *pb.TabletAlias, for wasAssigned := tablet.IsAssigned() tablet.Type = pb.TabletType_SCRAP // Update the tablet first, since that is canonical. - err = topo.UpdateTablet(ctx, ts, tablet) + err = ts.UpdateTablet(ctx, tablet) if err != nil { return err } @@ -125,5 +125,5 @@ func ChangeType(ctx context.Context, ts topo.Server, tabletAlias *pb.TabletAlias tablet.HealthMap = health } } - return topo.UpdateTablet(ctx, ts, tablet) + return ts.UpdateTablet(ctx, tablet) } diff --git a/go/vt/wrangler/tablet.go b/go/vt/wrangler/tablet.go index 6de86ce059..afa7aaf4a1 100644 --- a/go/vt/wrangler/tablet.go +++ b/go/vt/wrangler/tablet.go @@ -64,7 +64,7 @@ func (wr *Wrangler) InitTablet(ctx context.Context, tablet *pb.Tablet, force, cr } } - err := topo.CreateTablet(ctx, wr.ts, tablet) + err := wr.ts.CreateTablet(ctx, tablet) if err != nil && err == topo.ErrNodeExists { // Try to update nicely, but if it fails fall back to force behavior. if update || force { @@ -74,7 +74,7 @@ func (wr *Wrangler) InitTablet(ctx context.Context, tablet *pb.Tablet, force, cr } else { if oldTablet.Keyspace == tablet.Keyspace && oldTablet.Shard == tablet.Shard { *(oldTablet.Tablet) = *tablet - if err := topo.UpdateTablet(ctx, wr.ts, oldTablet); err != nil { + if err := wr.ts.UpdateTablet(ctx, oldTablet); err != nil { wr.Logger().Warningf("failed updating tablet %v: %v", topoproto.TabletAliasString(tablet.Alias), err) // now fall through the Scrap case } else { @@ -101,7 +101,7 @@ func (wr *Wrangler) InitTablet(ctx context.Context, tablet *pb.Tablet, force, cr // we ignore this wr.Logger().Errorf("failed deleting tablet %v: %v", topoproto.TabletAliasString(tablet.Alias), err) } - return topo.CreateTablet(ctx, wr.ts, tablet) + return wr.ts.CreateTablet(ctx, tablet) } } return err diff --git a/go/vt/zktopo/tablet.go b/go/vt/zktopo/tablet.go index fe5d298484..17123a03c7 100644 --- a/go/vt/zktopo/tablet.go +++ b/go/vt/zktopo/tablet.go @@ -80,7 +80,7 @@ func (zkts *Server) UpdateTablet(ctx context.Context, tablet *pb.Tablet, existin } // UpdateTabletFields is part of the topo.Server interface -func (zkts *Server) UpdateTabletFields(ctx context.Context, tabletAlias *pb.TabletAlias, update func(*pb.Tablet) error) error { +func (zkts *Server) UpdateTabletFields(ctx context.Context, tabletAlias *pb.TabletAlias, update func(*pb.Tablet) error) (*pb.Tablet, error) { // Store the last tablet value so we can log it if the change succeeds. var lastTablet *pb.Tablet @@ -109,16 +109,9 @@ func (zkts *Server) UpdateTabletFields(ctx context.Context, tabletAlias *pb.Tabl if zookeeper.IsError(err, zookeeper.ZNONODE) { err = topo.ErrNoNode } - return err + return nil, err } - - if lastTablet != nil { - event.Dispatch(&events.TabletChange{ - Tablet: *lastTablet, - Status: "updated", - }) - } - return nil + return lastTablet, nil } // DeleteTablet is part of the topo.Server interface From a9b45d1ea121bed151d1dad81c236cf90101811b Mon Sep 17 00:00:00 2001 From: Alain Jobart Date: Thu, 20 Aug 2015 10:56:59 -0700 Subject: [PATCH 79/80] Now fixing DeleteTablet, and adding event to CreateTablet. --- go/vt/etcdtopo/tablet.go | 21 +-------------------- go/vt/topo/tablet.go | 35 ++++++++++++++++++++++++++++++++++- go/vt/zktopo/tablet.go | 28 +--------------------------- 3 files changed, 36 insertions(+), 48 deletions(-) diff --git a/go/vt/etcdtopo/tablet.go b/go/vt/etcdtopo/tablet.go index d9cd7a9237..0685b73358 100644 --- a/go/vt/etcdtopo/tablet.go +++ b/go/vt/etcdtopo/tablet.go @@ -8,9 +8,7 @@ import ( "encoding/json" "fmt" - "github.com/youtube/vitess/go/event" "github.com/youtube/vitess/go/vt/topo" - "github.com/youtube/vitess/go/vt/topo/events" "github.com/youtube/vitess/go/vt/topo/topoproto" "golang.org/x/net/context" @@ -87,26 +85,9 @@ func (s *Server) DeleteTablet(ctx context.Context, tabletAlias *pb.TabletAlias) return err } - // Get the keyspace and shard names for the TabletChange event. - tablet, _, tErr := s.GetTablet(ctx, tabletAlias) - - _, err = cell.Delete(tabletDirPath(tabletAlias), true /* recursive */) - if err != nil { + if _, err = cell.Delete(tabletDirPath(tabletAlias), true /* recursive */); err != nil { return convertError(err) } - - // Only try to log if we have the required info. - if tErr == nil { - // Only copy the identity info for the tablet. The rest has been deleted. - event.Dispatch(&events.TabletChange{ - Tablet: pb.Tablet{ - Alias: tablet.Alias, - Keyspace: tablet.Keyspace, - Shard: tablet.Shard, - }, - Status: "deleted", - }) - } return nil } diff --git a/go/vt/topo/tablet.go b/go/vt/topo/tablet.go index ebb604fa14..9fc8d2e9ec 100644 --- a/go/vt/topo/tablet.go +++ b/go/vt/topo/tablet.go @@ -418,7 +418,40 @@ func (ts Server) CreateTablet(ctx context.Context, tablet *pb.Tablet) error { return nil } - return UpdateTabletReplicationData(ctx, ts, tablet) + if err := UpdateTabletReplicationData(ctx, ts, tablet); err != nil { + return err + } + + event.Dispatch(&events.TabletChange{ + Tablet: *tablet, + Status: "created", + }) + return nil +} + +// DeleteTablet wraps the underlying Impl.DeleteTablet +// and dispatches the event. +func (ts Server) DeleteTablet(ctx context.Context, tabletAlias *pb.TabletAlias) error { + // get the current tablet record, if any, to log the deletion + tablet, _, tErr := ts.Impl.GetTablet(ctx, tabletAlias) + + if err := ts.Impl.DeleteTablet(ctx, tabletAlias); err != nil { + return err + } + + // Only try to log if we have the required info. + if tErr == nil { + // Only copy the identity info for the tablet. The rest has been deleted. + event.Dispatch(&events.TabletChange{ + Tablet: pb.Tablet{ + Alias: tabletAlias, + Keyspace: tablet.Keyspace, + Shard: tablet.Shard, + }, + Status: "deleted", + }) + } + return nil } // UpdateTabletReplicationData creates or updates the replication diff --git a/go/vt/zktopo/tablet.go b/go/vt/zktopo/tablet.go index 17123a03c7..987ca90853 100644 --- a/go/vt/zktopo/tablet.go +++ b/go/vt/zktopo/tablet.go @@ -9,9 +9,7 @@ import ( "fmt" "sort" - "github.com/youtube/vitess/go/event" "github.com/youtube/vitess/go/vt/topo" - "github.com/youtube/vitess/go/vt/topo/events" "github.com/youtube/vitess/go/vt/topo/topoproto" "github.com/youtube/vitess/go/zk" "golang.org/x/net/context" @@ -50,11 +48,6 @@ func (zkts *Server) CreateTablet(ctx context.Context, tablet *pb.Tablet) error { } return err } - - event.Dispatch(&events.TabletChange{ - Tablet: *tablet, - Status: "created", - }) return nil } @@ -116,32 +109,13 @@ func (zkts *Server) UpdateTabletFields(ctx context.Context, tabletAlias *pb.Tabl // DeleteTablet is part of the topo.Server interface func (zkts *Server) DeleteTablet(ctx context.Context, alias *pb.TabletAlias) error { - // We need to find out the keyspace and shard names because - // those are required in the TabletChange event. - tablet, _, tiErr := zkts.GetTablet(ctx, alias) - zkTabletPath := TabletPathForAlias(alias) - err := zk.DeleteRecursive(zkts.zconn, zkTabletPath, -1) - if err != nil { + if err := zk.DeleteRecursive(zkts.zconn, zkTabletPath, -1); err != nil { if zookeeper.IsError(err, zookeeper.ZNONODE) { err = topo.ErrNoNode } return err } - - // Only try to log if we have the required information. - if tiErr == nil { - // We only want to copy the identity info for the tablet (alias, etc.). - // The rest has just been deleted, so it should be blank. - event.Dispatch(&events.TabletChange{ - Tablet: pb.Tablet{ - Alias: tablet.Alias, - Keyspace: tablet.Keyspace, - Shard: tablet.Shard, - }, - Status: "deleted", - }) - } return nil } From ddbbca247a033a5e0e2cd875f948bdc74ef4c70d Mon Sep 17 00:00:00 2001 From: Alain Jobart Date: Thu, 20 Aug 2015 12:20:56 -0700 Subject: [PATCH 80/80] Adding topo server setup to this test, required in some environments. --- test/python_client_test.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/python_client_test.py b/test/python_client_test.py index c26655dcc9..20c3d5d7b0 100755 --- a/test/python_client_test.py +++ b/test/python_client_test.py @@ -29,6 +29,8 @@ def setUpModule(): global vtgateclienttest_port global vtgateclienttest_grpc_port + environment.topo_server().setup() + vtgateclienttest_port = environment.reserve_ports(1) args = environment.binary_args('vtgateclienttest') + [ '-log_dir', environment.vtlogroot, @@ -49,6 +51,8 @@ def tearDownModule(): utils.kill_sub_process(vtgateclienttest_process, soft=True) vtgateclienttest_process.wait() + environment.topo_server().teardown() + class TestPythonClient(unittest.TestCase): CONNECT_TIMEOUT = 10.0