зеркало из https://github.com/github/vitess-gh.git
Mycnf cleanup - separated the code that reads it from that generates it. Standardized the
Mycnf struct and enforced correct initialization.
This commit is contained in:
Shruti Patil
Родитель
c94437f155
Коммит
d133a42d27
|
|
@ -1,9 +1,6 @@
|
|||
# master.cnf parameters
|
||||
|
||||
#master-info-file=file_name
|
||||
log-bin = {{.BinLogPath}}
|
||||
log-bin-index = {{.BinLogIndexPath}}
|
||||
log-slave-updates
|
||||
sync_binlog = 1
|
||||
# empty binlog-do-db seems to block replication
|
||||
#binlog-do-db = {{.DatabaseName}}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
relay-log = {{.RelayLogPath}}
|
||||
relay-log-index = {{.RelayLogIndexPath}}
|
||||
relay-log-info-file = {{.RelayLogInfoPath}}
|
||||
master-info-file = {{.MasterInfoFile}}
|
||||
|
||||
# required if this master is chained
|
||||
# probably safe to turn on all the time at the expense of some disk I/O
|
||||
|
|
@ -15,6 +16,3 @@ log-slave-updates
|
|||
# vt_enable_binlog_splitter_rbr = 1
|
||||
# vt_shard_key_range_start = {{.StartKey}}
|
||||
# vt_shard_key_range_end = {{.EndKey}}
|
||||
# vt_enable_send_cache_invalidation = 1
|
||||
# vt_host = {{.VtHost}} #localhost
|
||||
# vt_port = {{.VtPort}} # 6702
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import (
|
|||
"code.google.com/p/vitess/go/vt/dbconfigs"
|
||||
"code.google.com/p/vitess/go/vt/mysqlctl"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
)
|
||||
|
|
@ -16,7 +17,7 @@ import (
|
|||
var port = flag.Int("port", 6612, "vtocc port")
|
||||
var force = flag.Bool("force", false, "force action")
|
||||
var mysqlPort = flag.Int("mysql-port", 3306, "mysql port")
|
||||
var tabletUid = flag.Int("tablet-uid", 41983, "tablet uid")
|
||||
var tabletUid = flag.Uint("tablet-uid", 41983, "tablet uid")
|
||||
var logLevel = flag.String("log.level", "WARNING", "set log level")
|
||||
|
||||
func main() {
|
||||
|
|
@ -26,8 +27,8 @@ func main() {
|
|||
relog.LogNameToLogLevel(*logLevel))
|
||||
relog.SetLogger(logger)
|
||||
|
||||
vtRepl := mysqlctl.VtReplParams{TabletHost: "localhost", TabletPort: *port}
|
||||
mycnf := mysqlctl.NewMycnf(uint(*tabletUid), *mysqlPort, vtRepl)
|
||||
tabletAddr := fmt.Sprintf("%v:%v", "localhost", *port)
|
||||
mycnf := mysqlctl.NewMycnf(uint32(*tabletUid), *mysqlPort, mysqlctl.VtReplParams{})
|
||||
dbcfgs, err := dbconfigs.Init(mycnf)
|
||||
if err != nil {
|
||||
relog.Fatal("%s", err)
|
||||
|
|
@ -49,7 +50,7 @@ func main() {
|
|||
log.Fatalf("partialrestore failed: %v", err)
|
||||
}
|
||||
case "partialsnapshot":
|
||||
_, err := mysqld.CreateSplitReplicaSource(flag.Arg(1), flag.Arg(2), flag.Arg(3), flag.Arg(4), vtRepl.TabletAddr(), false)
|
||||
_, err := mysqld.CreateSplitReplicaSource(flag.Arg(1), flag.Arg(2), flag.Arg(3), flag.Arg(4), tabletAddr, false)
|
||||
if err != nil {
|
||||
log.Fatalf("partialsnapshot failed: %v", err)
|
||||
}
|
||||
|
|
@ -66,7 +67,7 @@ func main() {
|
|||
log.Fatalf("failed shutdown mysql: %v", mysqlErr)
|
||||
}
|
||||
case "snapshot":
|
||||
_, err := mysqld.CreateSnapshot(flag.Arg(1), vtRepl.TabletAddr(), false)
|
||||
_, err := mysqld.CreateSnapshot(flag.Arg(1), tabletAddr, false)
|
||||
if err != nil {
|
||||
log.Fatalf("snapshot failed: %v", err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ import (
|
|||
"os"
|
||||
"os/signal"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
|
|
@ -75,7 +76,14 @@ func main() {
|
|||
flag.Parse()
|
||||
|
||||
env.Init("vttablet")
|
||||
mycnf := readMycnf()
|
||||
|
||||
_, tabletidStr := path.Split(*tabletPath)
|
||||
tabletId, err := strconv.ParseUint(tabletidStr, 10, 32)
|
||||
if err != nil {
|
||||
relog.Fatal("Error converting tabletid to uint")
|
||||
}
|
||||
|
||||
mycnf := readMycnf(uint32(tabletId))
|
||||
dbcfgs, err := dbconfigs.Init(mycnf)
|
||||
if err != nil {
|
||||
relog.Warning("%s", err)
|
||||
|
|
@ -100,7 +108,7 @@ func main() {
|
|||
// NOTE: trailing slash in pattern means we handle all paths with this prefix
|
||||
// FIXME(msolomon) this path needs to be obtained from the config.
|
||||
http.Handle("/vt/snapshot/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
handleSnapshot(w, r, mycnf)
|
||||
handleSnapshot(w, r, mysqlctl.SnapshotDir(uint32(tabletId)))
|
||||
}))
|
||||
|
||||
// we delegate out startup to the micromanagement server so these actions
|
||||
|
|
@ -140,10 +148,9 @@ func serveRPC() {
|
|||
bsonrpc.ServeRPC()
|
||||
}
|
||||
|
||||
func readMycnf() *mysqlctl.Mycnf {
|
||||
func readMycnf(tabletId uint32) *mysqlctl.Mycnf {
|
||||
if *mycnfFile == "" {
|
||||
_, tabletid := path.Split(*tabletPath)
|
||||
*mycnfFile = fmt.Sprintf("%s/vt_%s/my.cnf", mysqlctl.VtDataRoot, tabletid)
|
||||
*mycnfFile = mysqlctl.MycnfFile(tabletId)
|
||||
}
|
||||
mycnf, mycnfErr := mysqlctl.ReadMycnf(*mycnfFile)
|
||||
if mycnfErr != nil {
|
||||
|
|
@ -214,9 +221,9 @@ func initQueryService(dbcfgs dbconfigs.DBConfigs) {
|
|||
})
|
||||
}
|
||||
|
||||
func handleSnapshot(rw http.ResponseWriter, req *http.Request, mycnf *mysqlctl.Mycnf) {
|
||||
func handleSnapshot(rw http.ResponseWriter, req *http.Request, snapshotDir string) {
|
||||
// FIXME(msolomon) some sort of security, no?
|
||||
if strings.HasPrefix(req.URL.Path, mycnf.SnapshotDir) {
|
||||
if strings.HasPrefix(req.URL.Path, snapshotDir) {
|
||||
relog.Info("serve %v", req.URL.Path)
|
||||
http.ServeFile(rw, req, req.URL.Path)
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -63,7 +63,7 @@ func main() {
|
|||
relog.LogNameToLogLevel(*logLevel))
|
||||
relog.SetLogger(logger)
|
||||
|
||||
zkConfig := zkctl.MakeZkConfigFromString(*zkCfg, *myId)
|
||||
zkConfig := zkctl.MakeZkConfigFromString(*zkCfg, uint32(*myId))
|
||||
zkd := zkctl.NewZkd(zkConfig)
|
||||
|
||||
action := flag.Arg(0)
|
||||
|
|
|
|||
|
|
@ -233,13 +233,13 @@ func (mysqld *Mysqld) CreateSnapshot(dbName, sourceAddr string, allowHierarchica
|
|||
}
|
||||
|
||||
var rs *ReplicaSource
|
||||
dataFiles, snapshotErr := mysqld.createSnapshot(dbName, mysqld.config.SnapshotDir)
|
||||
dataFiles, snapshotErr := mysqld.createSnapshot(dbName, mysqld.SnapshotDir)
|
||||
if snapshotErr != nil {
|
||||
relog.Error("CreateSnapshot failed: %v", snapshotErr)
|
||||
} else {
|
||||
rs = NewReplicaSource(sourceAddr, masterAddr, mysqld.replParams.Uname, mysqld.replParams.Pass,
|
||||
dbName, dataFiles, replicationPosition)
|
||||
rsFile := path.Join(mysqld.config.SnapshotDir, replicaSourceFile)
|
||||
rsFile := path.Join(mysqld.SnapshotDir, replicaSourceFile)
|
||||
if snapshotErr = writeJson(rsFile, rs); snapshotErr != nil {
|
||||
relog.Error("CreateSnapshot failed: %v", snapshotErr)
|
||||
}
|
||||
|
|
@ -341,7 +341,7 @@ func (mysqld *Mysqld) RestoreFromSnapshot(replicaSource *ReplicaSource) error {
|
|||
func (mysqld *Mysqld) fetchSnapshot(replicaSource *ReplicaSource) error {
|
||||
replicaDbPath := path.Join(mysqld.config.DataDir, replicaSource.DbName)
|
||||
|
||||
cleanDirs := []string{mysqld.config.SnapshotDir, replicaDbPath,
|
||||
cleanDirs := []string{mysqld.SnapshotDir, replicaDbPath,
|
||||
mysqld.config.InnodbDataHomeDir, mysqld.config.InnodbLogGroupHomeDir}
|
||||
|
||||
// clean out and start fresh
|
||||
|
|
@ -356,5 +356,5 @@ func (mysqld *Mysqld) fetchSnapshot(replicaSource *ReplicaSource) error {
|
|||
}
|
||||
}
|
||||
|
||||
return fetchFiles(replicaSource, mysqld.config.TabletDir)
|
||||
return fetchFiles(replicaSource, mysqld.TabletDir)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,143 +11,41 @@ package mysqlctl
|
|||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"os"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"text/template"
|
||||
)
|
||||
|
||||
type VtReplParams struct {
|
||||
TabletHost string
|
||||
TabletPort int
|
||||
StartKey string
|
||||
EndKey string
|
||||
}
|
||||
|
||||
func (vtrp VtReplParams) TabletAddr() string {
|
||||
return fmt.Sprintf("%v:%v", vtrp.TabletHost, vtrp.TabletPort)
|
||||
}
|
||||
|
||||
type Mycnf struct {
|
||||
ServerId uint
|
||||
TabletDir string
|
||||
SnapshotDir string
|
||||
ServerId uint32
|
||||
MysqlPort int
|
||||
DataDir string
|
||||
MycnfFile string
|
||||
InnodbDataHomeDir string
|
||||
InnodbLogGroupHomeDir string
|
||||
DatabaseName string // for replication FIXME(msolomon) should not be needed
|
||||
SocketFile string
|
||||
MysqlPort int
|
||||
VtHost string
|
||||
VtPort int
|
||||
StartKey string
|
||||
EndKey string
|
||||
ErrorLogPath string
|
||||
SlowLogPath string
|
||||
RelayLogPath string
|
||||
RelayLogIndexPath string
|
||||
RelayLogInfoPath string
|
||||
BinLogPath string
|
||||
BinLogIndexPath string
|
||||
MasterInfoFile string
|
||||
PidFile string
|
||||
mycnfMap map[string]string
|
||||
}
|
||||
|
||||
const (
|
||||
VtDataRoot = "/vt"
|
||||
snapshotDir = "snapshot"
|
||||
dataDir = "data"
|
||||
innodbDir = "innodb"
|
||||
relayLogDir = "relay-logs"
|
||||
binLogDir = "bin-logs"
|
||||
innodbDataSubdir = "innodb/data"
|
||||
innodbLogSubdir = "innodb/log"
|
||||
)
|
||||
|
||||
/* uid is a unique id for a particular tablet - it must be unique within the
|
||||
tabletservers deployed within a keyspace, lest there be collisions on disk.
|
||||
mysqldPort needs to be unique per instance per machine (shocking) but choosing
|
||||
this sensibly has nothing to do with the config, so I'll punt.
|
||||
*/
|
||||
func NewMycnf(uid uint, mysqlPort int, vtRepl VtReplParams) *Mycnf {
|
||||
cnf := new(Mycnf)
|
||||
cnf.ServerId = uid
|
||||
cnf.MysqlPort = mysqlPort
|
||||
cnf.TabletDir = fmt.Sprintf("%s/vt_%010d", VtDataRoot, uid)
|
||||
cnf.SnapshotDir = fmt.Sprintf("%s/%s/vt_%010d", VtDataRoot, snapshotDir, uid)
|
||||
cnf.DataDir = path.Join(cnf.TabletDir, dataDir)
|
||||
cnf.MycnfFile = path.Join(cnf.TabletDir, "my.cnf")
|
||||
cnf.InnodbDataHomeDir = path.Join(cnf.TabletDir, innodbDataSubdir)
|
||||
cnf.InnodbLogGroupHomeDir = path.Join(cnf.TabletDir, innodbLogSubdir)
|
||||
cnf.SocketFile = path.Join(cnf.TabletDir, "mysql.sock")
|
||||
cnf.VtHost = vtRepl.TabletHost
|
||||
cnf.VtPort = vtRepl.TabletPort
|
||||
cnf.StartKey = vtRepl.StartKey
|
||||
cnf.EndKey = vtRepl.EndKey
|
||||
return cnf
|
||||
}
|
||||
|
||||
func (cnf *Mycnf) TopLevelDirs() []string {
|
||||
return []string{dataDir, innodbDir, relayLogDir, binLogDir}
|
||||
}
|
||||
|
||||
func (cnf *Mycnf) DirectoryList() []string {
|
||||
return []string{
|
||||
cnf.DataDir,
|
||||
cnf.InnodbDataHomeDir,
|
||||
cnf.InnodbLogGroupHomeDir,
|
||||
cnf.relayLogDir(),
|
||||
cnf.binLogDir(),
|
||||
func (cnf *Mycnf) lookupAndCheck(key string) string {
|
||||
val := cnf.mycnfMap[key]
|
||||
if val == "" {
|
||||
panic(fmt.Errorf("Value for key '%v' not set", key))
|
||||
}
|
||||
}
|
||||
|
||||
func (cnf *Mycnf) ErrorLogPath() string {
|
||||
return path.Join(cnf.TabletDir, "error.log")
|
||||
}
|
||||
|
||||
func (cnf *Mycnf) SlowLogPath() string {
|
||||
return path.Join(cnf.TabletDir, "slow-query.log")
|
||||
}
|
||||
|
||||
func (cnf *Mycnf) relayLogDir() string {
|
||||
return path.Join(cnf.TabletDir, relayLogDir)
|
||||
}
|
||||
|
||||
func (cnf *Mycnf) RelayLogPath() string {
|
||||
return path.Join(cnf.relayLogDir(),
|
||||
fmt.Sprintf("vt-%010d-relay-bin", cnf.ServerId))
|
||||
}
|
||||
|
||||
func (cnf *Mycnf) RelayLogIndexPath() string {
|
||||
return cnf.RelayLogPath() + ".index"
|
||||
}
|
||||
|
||||
func (cnf *Mycnf) RelayLogInfoPath() string {
|
||||
return path.Join(cnf.TabletDir, "relay-logs", "relay.info")
|
||||
}
|
||||
|
||||
func (cnf *Mycnf) binLogDir() string {
|
||||
return path.Join(cnf.TabletDir, binLogDir)
|
||||
}
|
||||
|
||||
func (cnf *Mycnf) BinLogPath() string {
|
||||
return path.Join(cnf.binLogDir(),
|
||||
fmt.Sprintf("vt-%010d-bin", cnf.ServerId))
|
||||
}
|
||||
|
||||
func (cnf *Mycnf) BinLogPathForId(fileid int) string {
|
||||
return path.Join(cnf.binLogDir(),
|
||||
fmt.Sprintf("vt-%010d-bin.%06d", cnf.ServerId, fileid))
|
||||
}
|
||||
|
||||
func (cnf *Mycnf) BinLogIndexPath() string {
|
||||
return cnf.BinLogPath() + ".index"
|
||||
}
|
||||
|
||||
func (cnf *Mycnf) MasterInfoPath() string {
|
||||
return path.Join(cnf.TabletDir, "master.info")
|
||||
}
|
||||
|
||||
func (cnf *Mycnf) PidFile() string {
|
||||
return path.Join(cnf.TabletDir, "mysql.pid")
|
||||
return val
|
||||
}
|
||||
|
||||
func (cnf *Mycnf) MysqlAddr() string {
|
||||
|
|
@ -167,50 +65,13 @@ func fqdn() string {
|
|||
return strings.TrimRight(cname, ".")
|
||||
}
|
||||
|
||||
/*
|
||||
Join cnf files cnfPaths and subsitute in the right values.
|
||||
*/
|
||||
func MakeMycnf(cnfFiles []string, mycnf *Mycnf, header string) (string, error) {
|
||||
myTemplateSource := new(bytes.Buffer)
|
||||
for _, line := range strings.Split(header, "\n") {
|
||||
fmt.Fprintf(myTemplateSource, "## %v\n", strings.TrimSpace(line))
|
||||
}
|
||||
myTemplateSource.WriteString("[mysqld]\n")
|
||||
for _, path := range cnfFiles {
|
||||
data, dataErr := ioutil.ReadFile(path)
|
||||
if dataErr != nil {
|
||||
return "", dataErr
|
||||
func ReadMycnf(cnfFile string) (mycnf *Mycnf, err error) {
|
||||
defer func(err *error) {
|
||||
if x := recover(); x != nil {
|
||||
*err = x.(error)
|
||||
}
|
||||
myTemplateSource.WriteString("## " + path + "\n")
|
||||
myTemplateSource.Write(data)
|
||||
}
|
||||
}(&err)
|
||||
|
||||
myTemplate, err := template.New("").Parse(myTemplateSource.String())
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
mycnfData := new(bytes.Buffer)
|
||||
err = myTemplate.Execute(mycnfData, mycnf)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return mycnfData.String(), nil
|
||||
}
|
||||
|
||||
/* Create a config for this instance. Search cnfFiles for the appropriate
|
||||
cnf template files.
|
||||
*/
|
||||
func MakeMycnfForMysqld(mysqld *Mysqld, cnfFiles, header string) (string, error) {
|
||||
// FIXME(msolomon) determine config list from mysqld struct
|
||||
cnfs := []string{"default", "master", "replica"}
|
||||
paths := make([]string, len(cnfs))
|
||||
for i, name := range cnfs {
|
||||
paths[i] = fmt.Sprintf("%v/%v.cnf", cnfFiles, name)
|
||||
}
|
||||
return MakeMycnf(paths, mysqld.config, header)
|
||||
}
|
||||
|
||||
func ReadMycnf(cnfFile string) (*Mycnf, error) {
|
||||
f, err := os.Open(cnfFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -218,7 +79,10 @@ func ReadMycnf(cnfFile string) (*Mycnf, error) {
|
|||
defer f.Close()
|
||||
|
||||
buf := bufio.NewReader(f)
|
||||
mycnf := new(Mycnf)
|
||||
mycnf = new(Mycnf)
|
||||
mycnf.mycnfMap = make(map[string]string)
|
||||
var lval, rval string
|
||||
var parts [][]byte
|
||||
|
||||
for {
|
||||
line, _, err := buf.ReadLine()
|
||||
|
|
@ -226,37 +90,45 @@ func ReadMycnf(cnfFile string) (*Mycnf, error) {
|
|||
break
|
||||
}
|
||||
line = bytes.TrimSpace(line)
|
||||
if bytes.HasPrefix(line, []byte("server-id")) {
|
||||
serverId, err := strconv.Atoi(string(bytes.TrimSpace(bytes.Split(line, []byte("="))[1])))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("mycnf: failed to convert server-id %v", err)
|
||||
}
|
||||
mycnf.ServerId = uint(serverId)
|
||||
} else if bytes.HasPrefix(line, []byte("port")) {
|
||||
port, err := strconv.Atoi(string(bytes.TrimSpace(bytes.Split(line, []byte("="))[1])))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("mycnf: failed to convert port %v", err)
|
||||
}
|
||||
mycnf.MysqlPort = port
|
||||
} else if bytes.HasPrefix(line, []byte("datadir")) {
|
||||
mycnf.DataDir = string(bytes.TrimSpace(bytes.Split(line, []byte("="))[1]))
|
||||
} else if bytes.HasPrefix(line, []byte("innodb_log_group_home_dir")) {
|
||||
mycnf.InnodbLogGroupHomeDir = string(bytes.TrimSpace(bytes.Split(line, []byte("="))[1]))
|
||||
} else if bytes.HasPrefix(line, []byte("innodb_data_home_dir")) {
|
||||
mycnf.InnodbDataHomeDir = string(bytes.TrimSpace(bytes.Split(line, []byte("="))[1]))
|
||||
} else if bytes.HasPrefix(line, []byte("socket")) {
|
||||
mycnf.SocketFile = string(bytes.TrimSpace(bytes.Split(line, []byte("="))[1]))
|
||||
|
||||
parts = bytes.Split(line, []byte("="))
|
||||
if len(parts) < 2 {
|
||||
continue
|
||||
}
|
||||
lval = string(bytes.TrimSpace(parts[0]))
|
||||
rval = string(bytes.TrimSpace(parts[1]))
|
||||
mycnf.mycnfMap[lval] = rval
|
||||
}
|
||||
|
||||
// Make sure we run the correct initialization.
|
||||
vtMycnf := NewMycnf(mycnf.ServerId, mycnf.MysqlPort, VtReplParams{})
|
||||
serverIdStr := mycnf.lookupAndCheck("server-id")
|
||||
serverId, err := strconv.Atoi(serverIdStr)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("Failed to convert server-id %v", err))
|
||||
}
|
||||
mycnf.ServerId = uint32(serverId)
|
||||
|
||||
// Apply overrides.
|
||||
vtMycnf.DataDir = mycnf.DataDir
|
||||
vtMycnf.InnodbDataHomeDir = mycnf.InnodbDataHomeDir
|
||||
vtMycnf.InnodbLogGroupHomeDir = mycnf.InnodbLogGroupHomeDir
|
||||
vtMycnf.SocketFile = mycnf.SocketFile
|
||||
portStr := mycnf.lookupAndCheck("port")
|
||||
port, err := strconv.Atoi(portStr)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("Failed: failed to convert port %v", err))
|
||||
}
|
||||
mycnf.MysqlPort = port
|
||||
mycnf.DataDir = mycnf.lookupAndCheck("datadir")
|
||||
mycnf.InnodbDataHomeDir = mycnf.lookupAndCheck("innodb_data_home_dir")
|
||||
mycnf.InnodbLogGroupHomeDir = mycnf.lookupAndCheck("innodb_log_group_home_dir")
|
||||
mycnf.SocketFile = mycnf.lookupAndCheck("socket")
|
||||
mycnf.ErrorLogPath = mycnf.lookupAndCheck("log-error")
|
||||
mycnf.SlowLogPath = mycnf.lookupAndCheck("slow-query-log-file")
|
||||
mycnf.RelayLogPath = mycnf.lookupAndCheck("relay-log")
|
||||
mycnf.RelayLogIndexPath = mycnf.lookupAndCheck("relay-log-index")
|
||||
mycnf.RelayLogInfoPath = mycnf.lookupAndCheck("relay-log-info-file")
|
||||
mycnf.BinLogPath = mycnf.lookupAndCheck("log-bin")
|
||||
mycnf.BinLogIndexPath = mycnf.lookupAndCheck("log-bin-index")
|
||||
mycnf.MasterInfoFile = mycnf.lookupAndCheck("master-info-file")
|
||||
mycnf.PidFile = mycnf.lookupAndCheck("pid-file")
|
||||
//These values are currently not being set, hence not checking them.
|
||||
mycnf.StartKey = mycnf.mycnfMap["vt_shard_key_range_start"]
|
||||
mycnf.EndKey = mycnf.mycnfMap["vt_shard_key_range_end"]
|
||||
|
||||
return vtMycnf, nil
|
||||
return mycnf, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,136 @@
|
|||
// Copyright 2012, Google Inc. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
/*
|
||||
Generate my.cnf files from templates.
|
||||
*/
|
||||
|
||||
package mysqlctl
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"path"
|
||||
"strings"
|
||||
"text/template"
|
||||
)
|
||||
|
||||
type VtReplParams struct {
|
||||
StartKey string
|
||||
EndKey string
|
||||
}
|
||||
|
||||
const (
|
||||
VtDataRoot = "/vt"
|
||||
dataDir = "data"
|
||||
innodbDir = "innodb"
|
||||
relayLogDir = "relay-logs"
|
||||
binLogDir = "bin-logs"
|
||||
innodbDataSubdir = "innodb/data"
|
||||
innodbLogSubdir = "innodb/log"
|
||||
snapshotDir = "snapshot"
|
||||
)
|
||||
|
||||
/*
|
||||
NewMycnf fills the Mycnf structure with vt root paths and derived values.
|
||||
This is used to fill out the cnfTemplate values and generate my.cnf.
|
||||
uid is a unique id for a particular tablet - it must be unique within the
|
||||
tabletservers deployed within a keyspace, lest there be collisions on disk.
|
||||
mysqldPort needs to be unique per instance per machine.
|
||||
*/
|
||||
func NewMycnf(uid uint32, mysqlPort int, vtRepl VtReplParams) *Mycnf {
|
||||
cnf := new(Mycnf)
|
||||
tabletDir := TabletDir(uid)
|
||||
cnf.ServerId = uid
|
||||
cnf.MysqlPort = mysqlPort
|
||||
cnf.DataDir = path.Join(tabletDir, dataDir)
|
||||
cnf.InnodbDataHomeDir = path.Join(tabletDir, innodbDataSubdir)
|
||||
cnf.InnodbLogGroupHomeDir = path.Join(tabletDir, innodbLogSubdir)
|
||||
cnf.SocketFile = path.Join(tabletDir, "mysql.sock")
|
||||
cnf.StartKey = vtRepl.StartKey
|
||||
cnf.EndKey = vtRepl.EndKey
|
||||
cnf.ErrorLogPath = path.Join(tabletDir, "error.log")
|
||||
cnf.SlowLogPath = path.Join(tabletDir, "slow-query.log")
|
||||
cnf.RelayLogPath = path.Join(tabletDir, relayLogDir,
|
||||
fmt.Sprintf("vt-%010d-relay-bin", cnf.ServerId))
|
||||
cnf.RelayLogIndexPath = cnf.RelayLogPath + ".index"
|
||||
cnf.RelayLogInfoPath = path.Join(tabletDir, relayLogDir, "relay-log.info")
|
||||
cnf.BinLogPath = path.Join(tabletDir, binLogDir,
|
||||
fmt.Sprintf("vt-%010d-bin", cnf.ServerId))
|
||||
cnf.BinLogIndexPath = cnf.BinLogPath + ".index"
|
||||
cnf.MasterInfoFile = path.Join(tabletDir, "master.info")
|
||||
cnf.PidFile = path.Join(tabletDir, "mysql.pid")
|
||||
return cnf
|
||||
}
|
||||
|
||||
func TabletDir(uid uint32) string {
|
||||
return fmt.Sprintf("%s/vt_%010d", VtDataRoot, uid)
|
||||
}
|
||||
|
||||
func SnapshotDir(uid uint32) string {
|
||||
return fmt.Sprintf("%s/%s/vt_%010d", VtDataRoot, snapshotDir, uid)
|
||||
}
|
||||
|
||||
func MycnfFile(uid uint32) string {
|
||||
return path.Join(TabletDir(uid), "my.cnf")
|
||||
}
|
||||
|
||||
func TopLevelDirs() []string {
|
||||
return []string{dataDir, innodbDir, relayLogDir, binLogDir}
|
||||
}
|
||||
|
||||
func DirectoryList(cnf *Mycnf) []string {
|
||||
return []string{
|
||||
cnf.DataDir,
|
||||
cnf.InnodbDataHomeDir,
|
||||
cnf.InnodbLogGroupHomeDir,
|
||||
path.Join(TabletDir(cnf.ServerId), relayLogDir),
|
||||
path.Join(TabletDir(cnf.ServerId), binLogDir),
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
Join cnf files cnfPaths and subsitute in the right values.
|
||||
*/
|
||||
func MakeMycnf(cnfFiles []string, mycnf *Mycnf, header string) (string, error) {
|
||||
myTemplateSource := new(bytes.Buffer)
|
||||
for _, line := range strings.Split(header, "\n") {
|
||||
fmt.Fprintf(myTemplateSource, "## %v\n", strings.TrimSpace(line))
|
||||
}
|
||||
myTemplateSource.WriteString("[mysqld]\n")
|
||||
for _, path := range cnfFiles {
|
||||
data, dataErr := ioutil.ReadFile(path)
|
||||
if dataErr != nil {
|
||||
return "", dataErr
|
||||
}
|
||||
myTemplateSource.WriteString("## " + path + "\n")
|
||||
myTemplateSource.Write(data)
|
||||
}
|
||||
|
||||
myTemplate, err := template.New("").Parse(myTemplateSource.String())
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
mycnfData := new(bytes.Buffer)
|
||||
err = myTemplate.Execute(mycnfData, mycnf)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return mycnfData.String(), nil
|
||||
}
|
||||
|
||||
/* Create a config for this instance. Search cnfFiles for the appropriate
|
||||
cnf template files.
|
||||
*/
|
||||
func MakeMycnfForMysqld(mysqld *Mysqld, cnfFiles, header string) (string, error) {
|
||||
// FIXME(msolomon) determine config list from mysqld struct
|
||||
cnfs := []string{"default", "master", "replica"}
|
||||
paths := make([]string, len(cnfs))
|
||||
for i, name := range cnfs {
|
||||
paths[i] = fmt.Sprintf("%v/%v.cnf", cnfFiles, name)
|
||||
}
|
||||
return MakeMycnf(paths, mysqld.config, header)
|
||||
}
|
||||
|
|
@ -5,18 +5,19 @@
|
|||
package mysqlctl
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
var MYCNF_PATH = "/tmp/my.cnf"
|
||||
|
||||
func TestMycnf(t *testing.T) {
|
||||
var vtRepl VtReplParams
|
||||
vtRepl.TabletHost = "localhost"
|
||||
vtRepl.TabletPort = 6702
|
||||
vtRepl.StartKey = ""
|
||||
vtRepl.EndKey = ""
|
||||
|
||||
tablet0 := NewMysqld(NewMycnf(0, 6802, "", vtRepl), DefaultDbaParams, DefaultReplParams)
|
||||
tablet0 := NewMysqld(NewMycnf(0, 6802, vtRepl), DefaultDbaParams, DefaultReplParams)
|
||||
cnfTemplatePath := os.ExpandEnv("$VTROOT/src/code.google.com/p/vitess/config/mycnf")
|
||||
// FIXME(msolomon) make a path that has a chance of succeeding elsewhere
|
||||
data, err := MakeMycnfForMysqld(tablet0, cnfTemplatePath, "test header")
|
||||
|
|
@ -25,4 +26,19 @@ func TestMycnf(t *testing.T) {
|
|||
} else {
|
||||
t.Logf("data: %v", data)
|
||||
}
|
||||
err = ioutil.WriteFile(MYCNF_PATH, []byte(data), 0666)
|
||||
if err != nil {
|
||||
t.Errorf("failed creating my.cnf %v", err)
|
||||
}
|
||||
_, err = ioutil.ReadFile(MYCNF_PATH)
|
||||
if err != nil {
|
||||
t.Errorf("failed reading, err %v", err)
|
||||
return
|
||||
}
|
||||
mycnf, err := ReadMycnf(MYCNF_PATH)
|
||||
if err != nil {
|
||||
t.Errorf("failed reading, err %v", err)
|
||||
} else {
|
||||
t.Logf("socket file %v", mycnf.SocketFile)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -47,6 +47,9 @@ type Mysqld struct {
|
|||
dbaParams mysql.ConnectionParams
|
||||
replParams mysql.ConnectionParams
|
||||
createConnection CreateConnection
|
||||
TabletDir string
|
||||
SnapshotDir string
|
||||
MycnfFile string
|
||||
}
|
||||
|
||||
func NewMysqld(config *Mycnf, dba, repl mysql.ConnectionParams) *Mysqld {
|
||||
|
|
@ -57,8 +60,14 @@ func NewMysqld(config *Mycnf, dba, repl mysql.ConnectionParams) *Mysqld {
|
|||
createSuperConnection := func() (*mysql.Connection, error) {
|
||||
return mysql.Connect(dba)
|
||||
}
|
||||
|
||||
return &Mysqld{config, dba, repl, createSuperConnection}
|
||||
return &Mysqld{config,
|
||||
dba,
|
||||
repl,
|
||||
createSuperConnection,
|
||||
TabletDir(config.ServerId),
|
||||
SnapshotDir(config.ServerId),
|
||||
MycnfFile(config.ServerId),
|
||||
}
|
||||
}
|
||||
|
||||
func Start(mt *Mysqld) error {
|
||||
|
|
@ -66,7 +75,7 @@ func Start(mt *Mysqld) error {
|
|||
dir := os.ExpandEnv("$VT_MYSQL_ROOT")
|
||||
name := dir + "/bin/mysqld_safe"
|
||||
arg := []string{
|
||||
"--defaults-file=" + mt.config.MycnfFile}
|
||||
"--defaults-file=" + mt.MycnfFile}
|
||||
env := []string{
|
||||
os.ExpandEnv("LD_LIBRARY_PATH=$VT_MYSQL_ROOT/lib/mysql"),
|
||||
}
|
||||
|
|
@ -109,7 +118,7 @@ func Shutdown(mt *Mysqld, waitForMysqld bool) error {
|
|||
relog.Info("mysqlctl.Shutdown")
|
||||
// possibly mysql is already shutdown, check for a few files first
|
||||
_, socketPathErr := os.Stat(mt.config.SocketFile)
|
||||
_, pidPathErr := os.Stat(mt.config.PidFile())
|
||||
_, pidPathErr := os.Stat(mt.config.PidFile)
|
||||
if socketPathErr != nil && pidPathErr != nil {
|
||||
relog.Warning("assuming shutdown - no socket, no pid file")
|
||||
return nil
|
||||
|
|
@ -170,10 +179,10 @@ func Init(mt *Mysqld) error {
|
|||
cnfTemplatePath := os.ExpandEnv("$VTROOT/config/mycnf")
|
||||
configData, err := MakeMycnfForMysqld(mt, cnfTemplatePath, "tablet uid?")
|
||||
if err == nil {
|
||||
err = ioutil.WriteFile(mt.config.MycnfFile, []byte(configData), 0664)
|
||||
err = ioutil.WriteFile(mt.MycnfFile, []byte(configData), 0664)
|
||||
}
|
||||
if err != nil {
|
||||
relog.Error("failed creating %v: %v", mt.config.MycnfFile, err)
|
||||
relog.Error("failed creating %v: %v", mt.MycnfFile, err)
|
||||
return err
|
||||
}
|
||||
|
||||
|
|
@ -186,7 +195,7 @@ func Init(mt *Mysqld) error {
|
|||
return tarErr
|
||||
}
|
||||
if err = Start(mt); err != nil {
|
||||
relog.Error("failed starting, check %v", mt.config.ErrorLogPath())
|
||||
relog.Error("failed starting, check %v", mt.config.ErrorLogPath)
|
||||
return err
|
||||
}
|
||||
schemaPath := os.ExpandEnv("$VTROOT/data/bootstrap/_vt_schema.sql")
|
||||
|
|
@ -209,16 +218,16 @@ func Init(mt *Mysqld) error {
|
|||
}
|
||||
|
||||
func (mt *Mysqld) createDirs() error {
|
||||
relog.Info("creating directory %s", mt.config.TabletDir)
|
||||
if err := os.MkdirAll(mt.config.TabletDir, 0775); err != nil {
|
||||
relog.Info("creating directory %s", mt.TabletDir)
|
||||
if err := os.MkdirAll(mt.TabletDir, 0775); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, dir := range mt.config.TopLevelDirs() {
|
||||
for _, dir := range TopLevelDirs() {
|
||||
if err := mt.createTopDir(dir); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, dir := range mt.config.DirectoryList() {
|
||||
for _, dir := range DirectoryList(mt.config) {
|
||||
relog.Info("creating directory %s", dir)
|
||||
if err := os.MkdirAll(dir, 0775); err != nil {
|
||||
return err
|
||||
|
|
@ -236,19 +245,19 @@ func (mt *Mysqld) createDirs() error {
|
|||
// /vt/data/vt_xxxx
|
||||
// /vt/vt_xxxx/data -> /vt/data/vt_xxxx
|
||||
func (mt *Mysqld) createTopDir(dir string) error {
|
||||
vtname := path.Base(mt.config.TabletDir)
|
||||
vtname := path.Base(mt.TabletDir)
|
||||
target := path.Join(VtDataRoot, dir)
|
||||
_, err := os.Lstat(target)
|
||||
if err != nil {
|
||||
if err.(*os.PathError).Err == syscall.ENOENT {
|
||||
topdir := path.Join(mt.config.TabletDir, dir)
|
||||
topdir := path.Join(mt.TabletDir, dir)
|
||||
relog.Info("creating directory %s", topdir)
|
||||
return os.MkdirAll(topdir, 0775)
|
||||
}
|
||||
return err
|
||||
}
|
||||
linkto := path.Join(target, vtname)
|
||||
source := path.Join(mt.config.TabletDir, dir)
|
||||
source := path.Join(mt.TabletDir, dir)
|
||||
relog.Info("creating directory %s", linkto)
|
||||
err = os.MkdirAll(linkto, 0775)
|
||||
if err != nil {
|
||||
|
|
@ -267,8 +276,8 @@ func Teardown(mt *Mysqld, force bool) error {
|
|||
}
|
||||
}
|
||||
var removalErr error
|
||||
for _, dir := range mt.config.TopLevelDirs() {
|
||||
qdir := path.Join(mt.config.TabletDir, dir)
|
||||
for _, dir := range TopLevelDirs() {
|
||||
qdir := path.Join(mt.TabletDir, dir)
|
||||
if err := deleteTopDir(qdir); err != nil {
|
||||
removalErr = err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,8 +9,8 @@ import (
|
|||
)
|
||||
|
||||
func TestStartShutdown(t *testing.T) {
|
||||
mycnf0 := NewMycnf(0, 3700, "", VtReplParams{TabletHost: "localhost"})
|
||||
mycnf1 := NewMycnf(1, 3701, "", VtReplParams{TabletHost: "localhost"})
|
||||
mycnf0 := NewMycnf(0, 3700, VtReplParams{})
|
||||
mycnf1 := NewMycnf(1, 3701, VtReplParams{})
|
||||
tablet0 := NewMysqld(mycnf0, DefaultDbaParams, DefaultReplParams)
|
||||
tablet1 := NewMysqld(mycnf1, DefaultDbaParams, DefaultReplParams)
|
||||
var err error
|
||||
|
|
|
|||
|
|
@ -115,12 +115,12 @@ func StartReplicationCommands(replState *ReplicationState) []string {
|
|||
|
||||
// Read replication state from local files.
|
||||
func ReadReplicationState(mysqld *Mysqld) (*ReplicationState, error) {
|
||||
relayInfo, err := ioutil.ReadFile(mysqld.config.RelayLogInfoPath())
|
||||
relayInfo, err := ioutil.ReadFile(mysqld.config.RelayLogInfoPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// FIXME(msolomon) not sure i'll need this data
|
||||
masterInfo, err := ioutil.ReadFile(mysqld.config.MasterInfoPath())
|
||||
masterInfo, err := ioutil.ReadFile(mysqld.config.MasterInfoFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -166,7 +166,7 @@ func (mysqld *Mysqld) CreateSplitReplicaSource(dbName, keyName, startKey, endKey
|
|||
return
|
||||
}
|
||||
|
||||
cloneSourcePath := path.Join(mysqld.config.SnapshotDir, dataDir, dbName+"-"+b64ForFilename(startKey)+","+b64ForFilename(endKey))
|
||||
cloneSourcePath := path.Join(mysqld.SnapshotDir, dataDir, dbName+"-"+b64ForFilename(startKey)+","+b64ForFilename(endKey))
|
||||
// clean out and start fresh
|
||||
for _, _path := range []string{cloneSourcePath} {
|
||||
if err = os.RemoveAll(_path); err != nil {
|
||||
|
|
@ -262,7 +262,7 @@ func (mysqld *Mysqld) CreateSplitReplicaSource(dbName, keyName, startKey, endKey
|
|||
} else {
|
||||
rs = NewSplitReplicaSource(sourceAddr, masterAddr, mysqld.replParams.Uname, mysqld.replParams.Pass,
|
||||
dbName, dataFiles, replicationPosition, startKey, endKey, schema)
|
||||
rsFile := path.Join(mysqld.config.SnapshotDir, replicaSourceFile)
|
||||
rsFile := path.Join(mysqld.SnapshotDir, replicaSourceFile)
|
||||
if snapshotErr = writeJson(rsFile, rs); snapshotErr != nil {
|
||||
relog.Error("CreateSnapshot failed: %v", snapshotErr)
|
||||
}
|
||||
|
|
@ -360,7 +360,7 @@ func (mysqld *Mysqld) RestoreFromPartialSnapshot(replicaSource *SplitReplicaSour
|
|||
return
|
||||
}
|
||||
|
||||
tempStoragePath := path.Join(mysqld.config.SnapshotDir, "partialrestore")
|
||||
tempStoragePath := path.Join(mysqld.SnapshotDir, "partialrestore")
|
||||
cleanDirs := []string{tempStoragePath}
|
||||
|
||||
// clean out and start fresh
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ import (
|
|||
)
|
||||
|
||||
type zkServerAddr struct {
|
||||
ServerId uint
|
||||
ServerId uint32
|
||||
Hostname string
|
||||
LeaderPort int
|
||||
ElectionPort int
|
||||
|
|
@ -29,7 +29,7 @@ type zkServerAddr struct {
|
|||
}
|
||||
|
||||
type ZkConfig struct {
|
||||
ServerId uint
|
||||
ServerId uint32
|
||||
ClientPort int
|
||||
Servers []zkServerAddr
|
||||
Global bool
|
||||
|
|
@ -121,7 +121,7 @@ const GUESS_MYID = 0
|
|||
If server_id > 1000, then we assume this is a global quorum.
|
||||
server_id's must be 1-255, global id's are 1001-1255 mod 1000.
|
||||
*/
|
||||
func MakeZkConfigFromString(cmdLine string, myId uint) *ZkConfig {
|
||||
func MakeZkConfigFromString(cmdLine string, myId uint32) *ZkConfig {
|
||||
zkConfig := NewZkConfig()
|
||||
for _, zki := range strings.Split(cmdLine, ",") {
|
||||
zkiParts := strings.SplitN(zki, "@", 2)
|
||||
|
|
@ -137,7 +137,7 @@ func MakeZkConfigFromString(cmdLine string, myId uint) *ZkConfig {
|
|||
}
|
||||
myId = myId % 1000
|
||||
|
||||
zkServer := zkServerAddr{ServerId: uint(serverId), ClientPort: 2181,
|
||||
zkServer := zkServerAddr{ServerId: uint32(serverId), ClientPort: 2181,
|
||||
LeaderPort: 2888, ElectionPort: 3888}
|
||||
switch len(zkAddrParts) {
|
||||
case 4:
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче