2012-06-07 22:55:06 +04:00
|
|
|
// Copyright 2012, Google Inc. All rights reserved.
|
|
|
|
// Use of this source code is governed by a BSD-style
|
|
|
|
// license that can be found in the LICENSE file.
|
2012-02-25 11:30:03 +04:00
|
|
|
|
|
|
|
package tabletserver
|
|
|
|
|
|
|
|
import (
|
|
|
|
"fmt"
|
2014-04-04 22:40:43 +04:00
|
|
|
"strings"
|
2014-09-20 04:45:55 +04:00
|
|
|
"time"
|
2013-03-14 23:16:32 +04:00
|
|
|
|
Automatic rewrite of relog import paths and calls to use glog.
Commands run:
find go -name "*.go" | xargs sed --in-place -r 's,"github.com/youtube/vitess/go/relog",log "github.com/golang/glog",g; s,relog.Info,log.Infof,g; s,relog.Warning,log.Warningf,g; s,relog.Error,log.Errorf,g; s,relog.Fatal,log.Fatalf,g; s,relog.Debug,log.V(6).Infof,g'
find . -name '*.go' -exec gofmt -w {} \;
2013-08-07 01:56:00 +04:00
|
|
|
log "github.com/golang/glog"
|
2013-09-30 20:35:29 +04:00
|
|
|
"github.com/youtube/vitess/go/mysql"
|
2013-07-19 05:18:20 +04:00
|
|
|
"github.com/youtube/vitess/go/tb"
|
2014-09-20 04:45:55 +04:00
|
|
|
"github.com/youtube/vitess/go/vt/logutil"
|
2012-02-25 11:30:03 +04:00
|
|
|
)
|
|
|
|
|
|
|
|
const (
|
2015-01-06 20:37:34 +03:00
|
|
|
// ErrFail is returned when a query fails
|
|
|
|
ErrFail = iota
|
|
|
|
|
|
|
|
// ErrRetry is returned when a query can be retried
|
|
|
|
ErrRetry
|
|
|
|
|
|
|
|
// ErrFatal is returned when a query cannot be retried
|
|
|
|
ErrFatal
|
|
|
|
|
|
|
|
// ErrTxPoolFull is returned when we can't get a connection
|
|
|
|
ErrTxPoolFull
|
|
|
|
|
|
|
|
// ErrNotInTx is returned when we're not in a transaction but should be
|
|
|
|
ErrNotInTx
|
2012-02-25 11:30:03 +04:00
|
|
|
)
|
|
|
|
|
2014-09-20 04:45:55 +04:00
|
|
|
var logTxPoolFull = logutil.NewThrottledLogger("TxPoolFull", 1*time.Minute)
|
|
|
|
|
2015-01-06 20:37:34 +03:00
|
|
|
// TabletError is the erro type we use in this library
|
2012-02-25 11:30:03 +04:00
|
|
|
type TabletError struct {
|
|
|
|
ErrorType int
|
|
|
|
Message string
|
|
|
|
SqlError int
|
|
|
|
}
|
|
|
|
|
|
|
|
// This is how go-mysql exports its error number
|
|
|
|
type hasNumber interface {
|
|
|
|
Number() int
|
|
|
|
}
|
|
|
|
|
2015-01-06 20:37:34 +03:00
|
|
|
// NewTabletError returns a TabletError of the given type
|
2012-02-25 11:30:03 +04:00
|
|
|
func NewTabletError(errorType int, format string, args ...interface{}) *TabletError {
|
2014-04-04 22:40:43 +04:00
|
|
|
return &TabletError{
|
|
|
|
ErrorType: errorType,
|
|
|
|
Message: fmt.Sprintf(format, args...),
|
|
|
|
}
|
2012-02-25 11:30:03 +04:00
|
|
|
}
|
|
|
|
|
2015-01-06 20:37:34 +03:00
|
|
|
// NewTabletErrorSql returns a TabletError based on the error
|
2012-02-25 11:30:03 +04:00
|
|
|
func NewTabletErrorSql(errorType int, err error) *TabletError {
|
2014-04-04 22:40:43 +04:00
|
|
|
var errnum int
|
|
|
|
errstr := err.Error()
|
2012-02-25 11:30:03 +04:00
|
|
|
if sqlErr, ok := err.(hasNumber); ok {
|
2014-04-04 22:40:43 +04:00
|
|
|
errnum = sqlErr.Number()
|
|
|
|
// Override error type if MySQL is in read-only mode. It's probably because
|
|
|
|
// there was a remaster and there are old clients still connected.
|
2015-01-06 20:37:34 +03:00
|
|
|
if errnum == mysql.ErrOptionPreventsStatement && strings.Contains(errstr, "read-only") {
|
|
|
|
errorType = ErrRetry
|
2014-04-04 22:40:43 +04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
return &TabletError{
|
|
|
|
ErrorType: errorType,
|
|
|
|
Message: errstr,
|
|
|
|
SqlError: errnum,
|
2012-02-25 11:30:03 +04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-01-07 09:15:00 +03:00
|
|
|
func IsConnErr(err error) bool {
|
|
|
|
terr, ok := err.(*TabletError)
|
|
|
|
if !ok {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
// 2013 means that someone sniped the query.
|
|
|
|
if terr.SqlError == 2013 {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
return terr.SqlError >= 2000 && terr.SqlError <= 2018
|
|
|
|
}
|
|
|
|
|
2013-04-11 02:43:10 +04:00
|
|
|
func (te *TabletError) Error() string {
|
2012-02-25 11:30:03 +04:00
|
|
|
format := "error: %s"
|
2013-04-11 02:43:10 +04:00
|
|
|
switch te.ErrorType {
|
2015-01-06 20:37:34 +03:00
|
|
|
case ErrRetry:
|
2012-02-25 11:30:03 +04:00
|
|
|
format = "retry: %s"
|
2015-01-06 20:37:34 +03:00
|
|
|
case ErrFatal:
|
2012-02-25 11:30:03 +04:00
|
|
|
format = "fatal: %s"
|
2015-01-06 20:37:34 +03:00
|
|
|
case ErrTxPoolFull:
|
2013-08-21 01:00:00 +04:00
|
|
|
format = "tx_pool_full: %s"
|
2015-01-06 20:37:34 +03:00
|
|
|
case ErrNotInTx:
|
2013-09-21 09:21:30 +04:00
|
|
|
format = "not_in_tx: %s"
|
2012-02-25 11:30:03 +04:00
|
|
|
}
|
2013-04-11 02:43:10 +04:00
|
|
|
return fmt.Sprintf(format, te.Message)
|
2012-02-25 11:30:03 +04:00
|
|
|
}
|
|
|
|
|
2015-01-06 20:37:34 +03:00
|
|
|
// RecordStats will record the error in the proper stat bucket
|
2013-04-11 02:43:10 +04:00
|
|
|
func (te *TabletError) RecordStats() {
|
|
|
|
switch te.ErrorType {
|
2015-01-06 20:37:34 +03:00
|
|
|
case ErrRetry:
|
2013-12-08 08:16:52 +04:00
|
|
|
infoErrors.Add("Retry", 1)
|
2015-01-06 20:37:34 +03:00
|
|
|
case ErrFatal:
|
2013-12-08 08:16:52 +04:00
|
|
|
infoErrors.Add("Fatal", 1)
|
2015-01-06 20:37:34 +03:00
|
|
|
case ErrTxPoolFull:
|
2013-02-21 09:50:47 +04:00
|
|
|
errorStats.Add("TxPoolFull", 1)
|
2015-01-06 20:37:34 +03:00
|
|
|
case ErrNotInTx:
|
2013-09-21 09:21:30 +04:00
|
|
|
errorStats.Add("NotInTx", 1)
|
2012-02-25 11:30:03 +04:00
|
|
|
default:
|
2013-09-30 00:36:05 +04:00
|
|
|
switch te.SqlError {
|
2015-01-06 20:37:34 +03:00
|
|
|
case mysql.ErrDupEntry:
|
2013-12-08 08:16:52 +04:00
|
|
|
infoErrors.Add("DupKey", 1)
|
2015-01-06 20:37:34 +03:00
|
|
|
case mysql.ErrLockWaitTimeout, mysql.ErrLockDeadlock:
|
2013-09-30 00:36:05 +04:00
|
|
|
errorStats.Add("Deadlock", 1)
|
|
|
|
default:
|
2012-02-25 11:30:03 +04:00
|
|
|
errorStats.Add("Fail", 1)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-05-20 01:02:53 +04:00
|
|
|
func handleError(err *error, logStats *SQLQueryStats) {
|
2012-02-25 11:30:03 +04:00
|
|
|
if x := recover(); x != nil {
|
2012-12-20 05:48:02 +04:00
|
|
|
terr, ok := x.(*TabletError)
|
|
|
|
if !ok {
|
Automatic rewrite of relog import paths and calls to use glog.
Commands run:
find go -name "*.go" | xargs sed --in-place -r 's,"github.com/youtube/vitess/go/relog",log "github.com/golang/glog",g; s,relog.Info,log.Infof,g; s,relog.Warning,log.Warningf,g; s,relog.Error,log.Errorf,g; s,relog.Fatal,log.Fatalf,g; s,relog.Debug,log.V(6).Infof,g'
find . -name '*.go' -exec gofmt -w {} \;
2013-08-07 01:56:00 +04:00
|
|
|
log.Errorf("Uncaught panic:\n%v\n%s", x, tb.Stack(4))
|
2015-01-06 20:37:34 +03:00
|
|
|
*err = NewTabletError(ErrFail, "%v: uncaught panic", x)
|
2013-12-08 08:16:52 +04:00
|
|
|
internalErrors.Add("Panic", 1)
|
2012-12-20 05:48:02 +04:00
|
|
|
return
|
|
|
|
}
|
2012-02-25 11:30:03 +04:00
|
|
|
*err = terr
|
|
|
|
terr.RecordStats()
|
2015-01-06 20:37:34 +03:00
|
|
|
if terr.ErrorType == ErrRetry { // Retry errors are too spammy
|
2012-02-25 11:30:03 +04:00
|
|
|
return
|
|
|
|
}
|
2015-01-06 20:37:34 +03:00
|
|
|
if terr.ErrorType == ErrTxPoolFull {
|
2014-09-20 04:45:55 +04:00
|
|
|
logTxPoolFull.Errorf("%v", terr)
|
|
|
|
} else {
|
|
|
|
log.Errorf("%v", terr)
|
|
|
|
}
|
2012-02-25 11:30:03 +04:00
|
|
|
}
|
2014-11-19 11:25:40 +03:00
|
|
|
if logStats != nil {
|
|
|
|
logStats.Error = *err
|
|
|
|
logStats.Send()
|
|
|
|
}
|
2012-02-25 11:30:03 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
func logError() {
|
|
|
|
if x := recover(); x != nil {
|
2014-04-29 22:18:46 +04:00
|
|
|
terr, ok := x.(*TabletError)
|
|
|
|
if !ok {
|
|
|
|
log.Errorf("Uncaught panic:\n%v\n%s", x, tb.Stack(4))
|
|
|
|
internalErrors.Add("Panic", 1)
|
|
|
|
return
|
|
|
|
}
|
2015-01-06 20:37:34 +03:00
|
|
|
if terr.ErrorType == ErrTxPoolFull {
|
2014-09-20 04:45:55 +04:00
|
|
|
logTxPoolFull.Errorf("%v", terr)
|
|
|
|
} else {
|
|
|
|
log.Errorf("%v", terr)
|
|
|
|
}
|
2012-02-25 11:30:03 +04:00
|
|
|
}
|
|
|
|
}
|