improve error handling and path searching for zkctl

go/zk/zkctl/zkconf.go

@@ -78,21 +78,25 @@ func (cnf *ZkConfig) WriteMyid() error {
 }
 
 /*
-  Join cnf files cnfPaths and subsitute in the right values.
+  Search for first existing file in cnfFiles and subsitute in the right values.
 */
-func MakeZooCfg(cnfPaths []string, cnf *ZkConfig, header string) (string, error) {
+func MakeZooCfg(cnfFiles []string, cnf *ZkConfig, header string) (string, error) {
 	myTemplateSource := new(bytes.Buffer)
 	for _, line := range strings.Split(header, "\n") {
 		fmt.Fprintf(myTemplateSource, "## %v\n", strings.TrimSpace(line))
 	}
-	for _, path := range cnfPaths {
+	var dataErr error
+	for _, path := range cnfFiles {
 		data, dataErr := ioutil.ReadFile(path)
 		if dataErr != nil {
-			return "", dataErr
+			continue
 		}
 		myTemplateSource.WriteString("## " + path + "\n")
 		myTemplateSource.Write(data)
 	}
+	if dataErr != nil {
+		return "", dataErr
+	}
 
 	myTemplate, err := template.New("foo").Parse(myTemplateSource.String())
 	if err != nil {
@@ -108,19 +112,8 @@ func MakeZooCfg(cnfPaths []string, cnf *ZkConfig, header string) (string, error)
 
 const GUESS_MYID = 0
 
-func MakeZooCfgForString(cmdLine, cnfPath, header string) (string, error) {
-	cnfs := []string{"zoo"}
-	paths := make([]string, len(cnfs))
-	for i, name := range cnfs {
-		paths[i] = fmt.Sprintf("%v/%v.cfg", cnfPath, name)
-	}
-	zkConfig := MakeZkConfigFromString(cmdLine, GUESS_MYID)
-	return MakeZooCfg(paths, zkConfig, header)
-}
-
 /*
-  Create a config for this instance. Search cnfPath for the appropriate
-  cnf template files.
+  Create a config for this instance.
 
   <server_id>@<hostname>:<leader_port>:<election_port>:<client_port>
 
@@ -141,6 +134,8 @@ func MakeZkConfigFromString(cmdLine string, myId uint) *ZkConfig {
 			serverId = serverId % 1000
 			zkConfig.Global = true
 		}
+		myId = myId % 1000
+
 		zkServer := zkServerAddr{ServerId: uint(serverId), ClientPort: 2181,
 			LeaderPort: 2888, ElectionPort: 3888}
 		switch len(zkAddrParts) {

go/zk/zkctl/zkctl.go

@@ -58,8 +58,8 @@ func (zkd *Zkd) Start() error {
 	// NOTE(msolomon) use a script here so we can detach and continue to run
 	// if the wrangler process dies. this pretty much the same as mysqld_safe.
 	dirs := []string{
-		os.ExpandEnv("$VTROOT/dist/vt/bin"),
 		os.ExpandEnv("$VTROOT/src/code.google.com/p/vitess/go/zk/zkctl"),
+		os.ExpandEnv("/usr/local/bin"),
 	}
 
 	var name string
@@ -72,13 +72,13 @@ func (zkd *Zkd) Start() error {
 		}
 	}
 
-	arg := []string{
+	args := []string{
 		zkd.config.LogDir(),
 		zkd.config.ConfigFile(),
 		zkd.config.PidFile(),
 	}
 
-	cmd := exec.Command(name, arg...)
+	cmd := exec.Command(name, args...)
 	cmd.Env = os.Environ()
 	cmd.Dir = dir
 
@@ -88,18 +88,29 @@ func (zkd *Zkd) Start() error {
 
 	// give it some time to succeed - usually by the time the socket emerges
 	// we are in good shape
+	var err error
 	for i := 0; i < StartWaitTime; i++ {
-		conn, err := net.Dial("tcp", fmt.Sprintf(":%v", zkd.config.ClientPort))
-		if err != nil {
+		zkAddr := fmt.Sprintf(":%v", zkd.config.ClientPort)
+		conn, connErr := net.Dial("tcp", zkAddr)
+		if connErr != nil {
+			err = connErr
 			time.Sleep(1e9)
 			continue
 		} else {
+			err = nil
+			conn.Write([]byte("ruok"))
+			reply := make([]byte, 4)
+			conn.Read(reply)
+			if string(reply) != "imok" {
+				err = fmt.Errorf("local zk unhealthy: %v %v", zkAddr, reply)
+			}
 			conn.Close()
+			break
 		}
 	}
 	// wait so we don't get a bunch of defunct processes
 	go cmd.Wait()
-	return nil
+	return err
 }
 
 func (zkd *Zkd) Shutdown() error {
@@ -120,7 +131,10 @@ func (zkd *Zkd) Shutdown() error {
 }
 
 func (zkd *Zkd) makeCfg() (string, error) {
-	cnfTemplatePaths := []string{os.ExpandEnv("$VTROOT/src/code.google.com/p/vitess/config/zkcfg/zoo.cfg")}
+	cnfTemplatePaths := []string{
+		os.ExpandEnv("$VTROOT/src/code.google.com/p/vitess/config/zkcfg/zoo.cfg"),
+		os.ExpandEnv("/usr/local/share/vt/zkcfg/zoo.cfg"),
+	}
 	return MakeZooCfg(cnfTemplatePaths, zkd.config,
 		"# generated by vt")
 }
@@ -204,13 +218,13 @@ func (zkd *Zkd) init(preserveData bool) error {
 
 	if !preserveData {
 		_, err = zk.Create("/zk", "", 0, zookeeper.WorldACL(zookeeper.PERM_ALL))
-		if err != nil {
+		if err != nil && !zookeeper.IsError(err, zookeeper.ZNODEEXISTS) {
 			return err
 		}
 
 		if zkd.config.Global {
 			_, err = zk.Create("/zk/global", "", 0, zookeeper.WorldACL(zookeeper.PERM_ALL))
-			if err != nil {
+			if err != nil && !zookeeper.IsError(err, zookeeper.ZNODEEXISTS) {
 				return err
 			}
 		}
@@ -222,8 +236,7 @@ func (zkd *Zkd) init(preserveData bool) error {
 func (zkd *Zkd) Teardown() error {
 	relog.Info("zkctl.Teardown")
 	if err := zkd.Shutdown(); err != nil {
-		relog.Error("failed zookeeper shutdown: %v", err.Error())
-		return err
+		relog.Warning("failed zookeeper shutdown: %v", err.Error())
 	}
 	var removalErr error
 	for _, dir := range zkd.config.DirectoryList() {

go/zk/zkctl/zksrv.sh

@@ -11,8 +11,7 @@ logdir="$1"
 config="$2"
 pidfile="$3"
 
-zkroot="$VTROOT/dist/vt-zookeeper-3.3.2"
-classpath="$VTROOT/dist/vt-zookeeper-3.3.2/lib/zookeeper-3.3.2-fatjar.jar:$VTTOP/dist/vt-zookeeper-3.3.2/lib/"
+classpath="$VTROOT/dist/vt-zookeeper-3.3.2/lib/zookeeper-3.3.2-fatjar.jar:/usr/local/lib/zookeeper-3.3.2-fatjar.jar"
 
 mkdir -p "$logdir"
 touch "$logdir/zksrv.log"