queryservice: query rules extention

Add new action type for query rules to support RETRY errors.
Change table blacklist rule for vertical split to return RETRY errors.
This commit is contained in:
Sugu Sougoumarane 2014-05-05 15:41:37 -07:00
Родитель 659f018de1
Коммит 2fa5af21ac
5 изменённых файлов: 77 добавлений и 49 удалений

Просмотреть файл

@ -46,7 +46,7 @@ func (agent *ActionAgent) allowQueries(tablet *topo.Tablet) error {
// Compute the query rules that match the tablet record
qrs := tabletserver.LoadCustomRules()
if tablet.KeyRange.IsPartial() {
qr := tabletserver.NewQueryRule("enforce keyspace_id range", "keyspace_id_not_in_range", tabletserver.QR_FAIL_QUERY)
qr := tabletserver.NewQueryRule("enforce keyspace_id range", "keyspace_id_not_in_range", tabletserver.QR_FAIL)
qr.AddPlanCond(sqlparser.PLAN_INSERT_PK)
err := qr.AddBindVarCond("keyspace_id", true, true, tabletserver.QR_NOTIN, tablet.KeyRange)
if err != nil {
@ -57,7 +57,7 @@ func (agent *ActionAgent) allowQueries(tablet *topo.Tablet) error {
}
if len(tablet.BlacklistedTables) > 0 {
log.Infof("Blacklisting tables %v", strings.Join(tablet.BlacklistedTables, ", "))
qr := tabletserver.NewQueryRule("enforce blacklisted tables", "blacklisted_table", tabletserver.QR_FAIL_QUERY)
qr := tabletserver.NewQueryRule("enforce blacklisted tables", "blacklisted_table", tabletserver.QR_FAIL_RETRY)
for _, t := range tablet.BlacklistedTables {
qr.AddTableCond(t)
}

Просмотреть файл

@ -15,8 +15,8 @@ import (
func TestQueryRules(t *testing.T) {
qrs := NewQueryRules()
qr1 := NewQueryRule("rule 1", "r1", QR_FAIL_QUERY)
qr2 := NewQueryRule("rule 2", "r2", QR_FAIL_QUERY)
qr1 := NewQueryRule("rule 1", "r1", QR_FAIL)
qr2 := NewQueryRule("rule 2", "r2", QR_FAIL)
qrs.Add(qr1)
qrs.Add(qr2)
@ -51,12 +51,12 @@ func TestQueryRules(t *testing.T) {
// TestCopy tests for deep copy
func TestCopy(t *testing.T) {
qrs := NewQueryRules()
qr1 := NewQueryRule("rule 1", "r1", QR_FAIL_QUERY)
qr1 := NewQueryRule("rule 1", "r1", QR_FAIL)
qr1.AddPlanCond(sqlparser.PLAN_PASS_SELECT)
qr1.AddTableCond("aa")
qr1.AddBindVarCond("a", true, false, QR_NOOP, nil)
qr2 := NewQueryRule("rule 2", "r2", QR_FAIL_QUERY)
qr2 := NewQueryRule("rule 2", "r2", QR_FAIL)
qrs.Add(qr1)
qrs.Add(qr2)
@ -92,22 +92,22 @@ func TestCopy(t *testing.T) {
func TestFilterByPlan(t *testing.T) {
qrs := NewQueryRules()
qr1 := NewQueryRule("rule 1", "r1", QR_FAIL_QUERY)
qr1 := NewQueryRule("rule 1", "r1", QR_FAIL)
qr1.SetIPCond("123")
qr1.SetQueryCond("select")
qr1.AddPlanCond(sqlparser.PLAN_PASS_SELECT)
qr1.AddBindVarCond("a", true, false, QR_NOOP, nil)
qr2 := NewQueryRule("rule 2", "r2", QR_FAIL_QUERY)
qr2 := NewQueryRule("rule 2", "r2", QR_FAIL)
qr2.AddPlanCond(sqlparser.PLAN_PASS_SELECT)
qr2.AddPlanCond(sqlparser.PLAN_PK_EQUAL)
qr2.AddBindVarCond("a", true, false, QR_NOOP, nil)
qr3 := NewQueryRule("rule 3", "r3", QR_FAIL_QUERY)
qr3 := NewQueryRule("rule 3", "r3", QR_FAIL)
qr3.SetQueryCond("sele.*")
qr3.AddBindVarCond("a", true, false, QR_NOOP, nil)
qr4 := NewQueryRule("rule 4", "r4", QR_FAIL_QUERY)
qr4 := NewQueryRule("rule 4", "r4", QR_FAIL)
qr4.AddTableCond("b")
qr4.AddTableCond("c")
@ -167,7 +167,7 @@ func TestFilterByPlan(t *testing.T) {
t.Errorf("want r5, got %s", qrs1.rules[0].Name)
}
qr5 := NewQueryRule("rule 5", "r5", QR_FAIL_QUERY)
qr5 := NewQueryRule("rule 5", "r5", QR_FAIL)
qrs.Add(qr5)
qrs1 = qrs.filterByPlan("sel", sqlparser.PLAN_INSERT_PK, "a")
@ -185,7 +185,7 @@ func TestFilterByPlan(t *testing.T) {
}
func TestQueryRule(t *testing.T) {
qr := NewQueryRule("rule 1", "r1", QR_FAIL_QUERY)
qr := NewQueryRule("rule 1", "r1", QR_FAIL)
err := qr.SetIPCond("123")
if err != nil {
t.Errorf("unexpected: %v", err)
@ -221,7 +221,7 @@ func TestQueryRule(t *testing.T) {
}
func TestBindVarStruct(t *testing.T) {
qr := NewQueryRule("rule 1", "r1", QR_FAIL_QUERY)
qr := NewQueryRule("rule 1", "r1", QR_FAIL)
var err error
err = qr.AddBindVarCond("b", false, true, QR_NOOP, nil)
@ -294,7 +294,7 @@ var creationCases = []BVCreation{
}
func TestBVCreation(t *testing.T) {
qr := NewQueryRule("rule 1", "r1", QR_FAIL_QUERY)
qr := NewQueryRule("rule 1", "r1", QR_FAIL)
for i, tcase := range creationCases {
err := qr.AddBindVarCond(tcase.name, tcase.onAbsent, tcase.onMismatch, tcase.op, tcase.value)
haserr := (err != nil)
@ -473,13 +473,13 @@ func TestBVConditions(t *testing.T) {
func TestAction(t *testing.T) {
qrs := NewQueryRules()
qr1 := NewQueryRule("rule 1", "r1", QR_FAIL_QUERY)
qr1 := NewQueryRule("rule 1", "r1", QR_FAIL)
qr1.SetIPCond("123")
qr2 := NewQueryRule("rule 2", "r2", QR_FAIL_QUERY)
qr2 := NewQueryRule("rule 2", "r2", QR_FAIL_RETRY)
qr2.SetUserCond("user")
qr3 := NewQueryRule("rule 3", "r3", QR_FAIL_QUERY)
qr3 := NewQueryRule("rule 3", "r3", QR_FAIL)
qr3.AddBindVarCond("a", true, true, QR_EQ, uint64(1))
qrs.Add(qr1)
@ -489,15 +489,15 @@ func TestAction(t *testing.T) {
bv := make(map[string]interface{})
bv["a"] = uint64(0)
action, desc := qrs.getAction("123", "user1", bv)
if action != QR_FAIL_QUERY {
if action != QR_FAIL {
t.Errorf("want fail")
}
if desc != "rule 1" {
t.Errorf("want rule 1, got %s", desc)
}
action, desc = qrs.getAction("1234", "user", bv)
if action != QR_FAIL_QUERY {
t.Errorf("want fail")
if action != QR_FAIL_RETRY {
t.Errorf("want fail_retry")
}
if desc != "rule 2" {
t.Errorf("want rule 2, got %s", desc)
@ -508,7 +508,7 @@ func TestAction(t *testing.T) {
}
bv["a"] = uint64(1)
action, desc = qrs.getAction("1234", "user1", bv)
if action != QR_FAIL_QUERY {
if action != QR_FAIL {
t.Errorf("want fail")
}
if desc != "rule 3" {
@ -534,7 +534,8 @@ var jsondata = `[{
"OnMismatch": true,
"Operator": "UEQ",
"Value": "123"
}]
}],
"Action": "FAIL_RETRY"
},{
"Description": "desc2",
"Name": "name2"
@ -594,6 +595,9 @@ func TestImport(t *testing.T) {
if !bvc.onMismatch {
t.Errorf("want true")
}
if qrs.rules[0].act != QR_FAIL_RETRY {
t.Errorf("want FAIL_RETRY")
}
if bvc.op != QR_EQ {
t.Errorf("want %v, got %v", QR_EQ, bvc.op)
}
@ -621,6 +625,9 @@ func TestImport(t *testing.T) {
if qrs.rules[1].bindVarConds != nil {
t.Errorf("want nil")
}
if qrs.rules[1].act != QR_FAIL {
t.Errorf("want FAIL")
}
}
type ValidJSONCase struct {
@ -715,11 +722,11 @@ var invalidjsons = []InvalidJSONCase{
{`[{"Plans": 1 }]`, "want list for Plans"},
{`[{"TableNames": 1 }]`, "want list for TableNames"},
{`[{"BindVarConds": 1 }]`, "want list for BindVarConds"},
{`[{"RequestIP": "[" }]`, "Could not set IP condition: ["},
{`[{"User": "[" }]`, "Could not set User condition: ["},
{`[{"Query": "[" }]`, "Could not set Query condition: ["},
{`[{"RequestIP": "[" }]`, "could not set IP condition: ["},
{`[{"User": "[" }]`, "could not set User condition: ["},
{`[{"Query": "[" }]`, "could not set Query condition: ["},
{`[{"Plans": [1] }]`, "want string for Plans"},
{`[{"Plans": ["invalid"] }]`, "Invalid plan name: invalid"},
{`[{"Plans": ["invalid"] }]`, "invalid plan name: invalid"},
{`[{"TableNames": [1] }]`, "want string for TableNames"},
{`[{"BindVarConds": [1] }]`, "want json object for bind var conditions"},
{`[{"BindVarConds": [{}] }]`, "Name missing in BindVarConds"},
@ -727,7 +734,7 @@ var invalidjsons = []InvalidJSONCase{
{`[{"BindVarConds": [{"Name": "a"}] }]`, "OnAbsent missing in BindVarConds"},
{`[{"BindVarConds": [{"Name": "a", "OnAbsent": 1}] }]`, "want bool for OnAbsent"},
{`[{"BindVarConds": [{"Name": "a", "OnAbsent": true}]}]`, "Operator missing in BindVarConds"},
{`[{"BindVarConds": [{"Name": "a", "OnAbsent": true, "Operator": "a"}]}]`, "Invalid Operator a"},
{`[{"BindVarConds": [{"Name": "a", "OnAbsent": true, "Operator": "a"}]}]`, "invalid Operator a"},
{`[{"BindVarConds": [{"Name": "a", "OnAbsent": true, "Operator": "UEQ"}]}]`, "Value missing in BindVarConds"},
{`[{"BindVarConds": [{"Name": "a", "OnAbsent": true, "Operator": "UEQ", "Value": "a"}]}]`, "want uint64: a"},
@ -761,16 +768,18 @@ var invalidjsons = []InvalidJSONCase{
{`[{"BindVarConds": [{"Name": "a", "OnAbsent": true, "Operator": "ILE", "Value": "1"}]}]`, "OnMismatch missing in BindVarConds"},
{`[{"BindVarConds": [{"Name": "a", "OnAbsent": true, "OnMismatch": true, "Operator": "MATCH", "Value": "["}]}]`, "Processing [: error parsing regexp: missing closing ]: `[$`"},
{`[{"BindVarConds": [{"Name": "a", "OnAbsent": true, "OnMismatch": true, "Operator": "NOMATCH", "Value": "["}]}]`, "Processing [: error parsing regexp: missing closing ]: `[$`"},
{`[{"BindVarConds": [{"Name": "a", "OnAbsent": true, "OnMismatch": true, "Operator": "MATCH", "Value": "["}]}]`, "processing [: error parsing regexp: missing closing ]: `[$`"},
{`[{"BindVarConds": [{"Name": "a", "OnAbsent": true, "OnMismatch": true, "Operator": "NOMATCH", "Value": "["}]}]`, "processing [: error parsing regexp: missing closing ]: `[$`"},
{`[{"Action": 1 }]`, "want string for Action"},
{`[{"Action": "foo" }]`, "invalid Action foo"},
}
func TestInvalidJSON(t *testing.T) {
for i, tcase := range invalidjsons {
for _, tcase := range invalidjsons {
qrs := NewQueryRules()
err := qrs.UnmarshalJSON([]byte(tcase.input))
if err == nil {
t.Errorf("want error for case %d", i)
t.Errorf("want error for case %q", tcase.input)
continue
}
recvd := strings.Replace(err.Error(), "error: ", "", 1)

Просмотреть файл

@ -263,8 +263,11 @@ func (qe *QueryEngine) Execute(logStats *sqlQueryStats, query *proto.Query) (rep
// Run it by the rules engine
action, desc := basePlan.Rules.getAction(logStats.RemoteAddr(), logStats.Username(), query.BindVariables)
if action == QR_FAIL_QUERY {
switch action {
case QR_FAIL:
panic(NewTabletError(FAIL, "Query disallowed due to rule: %s", desc))
case QR_FAIL_RETRY:
panic(NewTabletError(RETRY, "Query disallowed due to rule: %s", desc))
}
if basePlan.PlanId == sqlparser.PLAN_DDL {

Просмотреть файл

@ -102,8 +102,8 @@ func (qrs *QueryRules) filterByPlan(query string, planid sqlparser.PlanType, tab
func (qrs *QueryRules) getAction(ip, user string, bindVars map[string]interface{}) (action Action, desc string) {
for _, qr := range qrs.rules {
if qr.getAction(ip, user, bindVars) == QR_FAIL_QUERY {
return QR_FAIL_QUERY, qr.Description
if act := qr.getAction(ip, user, bindVars); act != QR_CONTINUE {
return act, qr.Description
}
}
return QR_CONTINUE, ""
@ -136,12 +136,15 @@ type QueryRule struct {
// All BindVar conditions have to be fulfilled to make this true (AND)
bindVarConds []BindVarCond
// Action to be performed on trigger
act Action
}
// NewQueryRule creates a new QueryRule.
func NewQueryRule(description, name string, act Action) (qr *QueryRule) {
// We ignore act because there's only one action right now
return &QueryRule{Description: description, Name: name}
return &QueryRule{Description: description, Name: name, act: act}
}
// Copy performs a deep copy of a QueryRule.
@ -152,6 +155,7 @@ func (qr *QueryRule) Copy() (newqr *QueryRule) {
requestIP: qr.requestIP,
user: qr.user,
query: qr.query,
act: qr.act,
}
if qr.plans != nil {
newqr.plans = make([]sqlparser.PlanType, len(qr.plans))
@ -250,7 +254,7 @@ func (qr *QueryRule) AddBindVarCond(name string, onAbsent, onMismatch bool, op O
// Change the value to compiled regexp
re, err := regexp.Compile(makeExact(v))
if err != nil {
return NewTabletError(FAIL, "Processing %s: %v", v, err)
return NewTabletError(FAIL, "processing %s: %v", v, err)
}
converted = bvcre{re}
} else {
@ -262,13 +266,13 @@ func (qr *QueryRule) AddBindVarCond(name string, onAbsent, onMismatch bool, op O
}
converted = bvcKeyRange(v)
default:
return NewTabletError(FAIL, "Type %T not allowed as condition operand (%v)", value, value)
return NewTabletError(FAIL, "type %T not allowed as condition operand (%v)", value, value)
}
qr.bindVarConds = append(qr.bindVarConds, BindVarCond{name, onAbsent, onMismatch, op, converted})
return nil
Error:
return NewTabletError(FAIL, "Invalid operator %s for type %T (%v)", op, value, value)
return NewTabletError(FAIL, "invalid operator %s for type %T (%v)", op, value, value)
}
// filterByPlan returns a new QueryRule if the query and planid match.
@ -304,7 +308,7 @@ func (qr *QueryRule) getAction(ip, user string, bindVars map[string]interface{})
return QR_CONTINUE
}
}
return QR_FAIL_QUERY
return qr.act
}
func reMatch(re *regexp.Regexp, val string) bool {
@ -353,8 +357,11 @@ func bvMatch(bvcond BindVarCond, bindVars map[string]interface{}) bool {
// when a QueryRule is triggered.
type Action int
const QR_CONTINUE = Action(0)
const QR_FAIL_QUERY = Action(1)
const (
QR_CONTINUE = Action(iota)
QR_FAIL
QR_FAIL_RETRY
)
// BindVarCond represents a bind var condition.
type BindVarCond struct {
@ -676,13 +683,13 @@ func getstring(val interface{}) (sv string, status int) {
// Support functions for JSON
func buildQueryRule(ruleInfo map[string]interface{}) (qr *QueryRule, err error) {
qr = NewQueryRule("", "", QR_FAIL_QUERY)
qr = NewQueryRule("", "", QR_FAIL)
for k, v := range ruleInfo {
var sv string
var lv []interface{}
var ok bool
switch k {
case "Name", "Description", "RequestIP", "User", "Query":
case "Name", "Description", "RequestIP", "User", "Query", "Action":
sv, ok = v.(string)
if !ok {
return nil, NewTabletError(FAIL, "want string for %s", k)
@ -703,17 +710,17 @@ func buildQueryRule(ruleInfo map[string]interface{}) (qr *QueryRule, err error)
case "RequestIP":
err = qr.SetIPCond(sv)
if err != nil {
return nil, NewTabletError(FAIL, "Could not set IP condition: %v", sv)
return nil, NewTabletError(FAIL, "could not set IP condition: %v", sv)
}
case "User":
err = qr.SetUserCond(sv)
if err != nil {
return nil, NewTabletError(FAIL, "Could not set User condition: %v", sv)
return nil, NewTabletError(FAIL, "could not set User condition: %v", sv)
}
case "Query":
err = qr.SetQueryCond(sv)
if err != nil {
return nil, NewTabletError(FAIL, "Could not set Query condition: %v", sv)
return nil, NewTabletError(FAIL, "could not set Query condition: %v", sv)
}
case "Plans":
for _, p := range lv {
@ -723,7 +730,7 @@ func buildQueryRule(ruleInfo map[string]interface{}) (qr *QueryRule, err error)
}
pt, ok := sqlparser.PlanByName(pv)
if !ok {
return nil, NewTabletError(FAIL, "Invalid plan name: %s", pv)
return nil, NewTabletError(FAIL, "invalid plan name: %s", pv)
}
qr.AddPlanCond(pt)
}
@ -747,6 +754,15 @@ func buildQueryRule(ruleInfo map[string]interface{}) (qr *QueryRule, err error)
return nil, err
}
}
case "Action":
switch sv {
case "FAIL":
qr.act = QR_FAIL
case "FAIL_RETRY":
qr.act = QR_FAIL_RETRY
default:
return nil, NewTabletError(FAIL, "invalid Action %s", sv)
}
}
}
return qr, nil
@ -794,7 +810,7 @@ func buildBindVarCondition(bvc interface{}) (name string, onAbsent, onMismatch b
}
op, ok = opmap[strop]
if !ok {
err = NewTabletError(FAIL, "Invalid Operator %s", strop)
err = NewTabletError(FAIL, "invalid Operator %s", strop)
return
}
if op == QR_NOOP {

Просмотреть файл

@ -208,7 +208,7 @@ index by_msg (msg)
# table is blacklisted, should get the error
out, err = tablet.vquery("select count(1) from %s" % t,
path='source_keyspace/0', raise_on_error=False)
self.assertTrue(err.find("Query disallowed due to rule: enforce blacklisted tables") != -1, "Cannot find the right error message in query for blacklisted table: out=\n%serr=\n%s" % (out, err))
self.assertTrue(err.find("retry: Query disallowed due to rule: enforce blacklisted tables") != -1, "Cannot find the right error message in query for blacklisted table: out=\n%serr=\n%s" % (out, err))
else:
# table is not blacklisted, should just work
tablet.vquery("select count(1) from %s" % t, path='source_keyspace/0')