Adding status page fragment for vttablet health.

go/cmd/vttablet/status.go

@@ -0,0 +1,96 @@
+package main
+
+import (
+	"html/template"
+
+	"github.com/youtube/vitess/go/vt/health"
+	"github.com/youtube/vitess/go/vt/servenv"
+	"github.com/youtube/vitess/go/vt/tabletmanager"
+)
+
+var (
+	healthTemplate = `
+<style>
+  table {
+    width: 70%;
+    border-collapse: collapse;
+  }
+  td, th {
+    border: 1px solid #999;
+    padding: 0.5rem;
+  }
+  .time {
+    width: 15%;
+  }
+  .healthy {
+    background-color: LightGreen;
+  }
+  .unhealthy {
+    background-color: Salmon;
+  }
+  .unhappy {
+    background-color: Khaki;
+  }
+</style>
+<div style="font-size: x-large">Current status: <span style="padding-left: 0.5em; padding-right: 0.5em; padding-bottom: 0.5ex; padding-top: 0.5ex;" class="{{.Current}}">{{.Current}}</span></div>
+<p>Polling health information from {{github_com_youtube_vitess_health_html_name}}.</p>
+<h2>History</h2>
+<table>
+  <tr>
+    <th class="time">time</th>
+    <th>state</th>
+  </tr>
+  {{range .Records}}
+  <tr class="{{.Class}}">
+    <td class="time">{{.Time.Format "Jan 2, 2006 at 15:04:05 (MST)"}}</td>
+    <td>
+    {{ if eq "unhealthy" .Class}}
+      unhealthy: {{.Error}}
+    {{else if eq "unhappy" .Class}}
+      unhappy (reasons: {{range $key, $value := .Result}}{{$key}}: {{$value}} {{end}})
+    {{else}}
+      healthy
+    {{end}}
+  </tr>
+  {{end}}
+</table>
+<dl style="font-size: small;">
+  <dt><span class="healthy">healthy</span></dt>
+  <dd>serving traffic.</dd>
+
+  <dt><span class="unhappy">unhappy</span></dt>
+  <dd>will serve traffic only if there are no fully healthy tablets.</dd>
+
+  <dt><span class="unhealthy">unhealthy</span></dt>
+  <dd>will not serve traffic.</dd>
+</dl>
+`
+)
+
+type healthStatus struct {
+	Records []interface{}
+}
+
+func (hs *healthStatus) Current() string {
+	if len(hs.Records) > 0 {
+		return hs.Records[0].(*tabletmanager.HealthRecord).Class()
+	}
+	return "unknown"
+}
+
+func healthHTMLName() string {
+	return health.HTMLName()
+}
+
+func init() {
+	servenv.OnRun(func() {
+		if agent.IsRunningHealthCheck() {
+			servenv.AddStatusFuncs(template.FuncMap{
+				"github_com_youtube_vitess_health_html_name": healthHTMLName,
+			})
+			servenv.AddStatusPart("Health", healthTemplate, func() interface{} {
+				return &healthStatus{Records: agent.History.Records()}
+			})
+		}
+	})
+}

go/vt/health/health.go

@@ -2,6 +2,7 @@ package health
 
 import (
 	"fmt"
+	"strings"
 	"sync"
 
 	log "github.com/golang/glog"
@@ -34,15 +35,25 @@ type Reporter interface {
 	// error it implies that the tablet is in a bad shape and not
 	// able to handle queries.
 	Report(typ topo.TabletType) (status map[string]string, err error)
+
+	// HTMLName returns a displayable name for the module.
+	// Can be used to be displayed in the status page.
+	HTMLName() string
 }
 
 // FunctionReporter is a function that may act as a Reporter.
 type FunctionReporter func(typ topo.TabletType) (map[string]string, error)
 
+// Report implements Reporter.Report
 func (fc FunctionReporter) Report(typ topo.TabletType) (status map[string]string, err error) {
 	return fc(typ)
 }
 
+// HTMLName implements Reporter.HTMLName
+func (fc FunctionReporter) HTMLName() string {
+	return "FunctionReporter"
+}
+
 // Aggregator aggregates the results of many Reporters.
 type Aggregator struct {
 	// mu protects all fields below its declaration.
@@ -50,6 +61,7 @@ type Aggregator struct {
 	reporters map[string]Reporter
 }
 
+// NewAggregator returns a new empty Aggregator
 func NewAggregator() *Aggregator {
 	return &Aggregator{
 		reporters: make(map[string]Reporter),
@@ -116,6 +128,17 @@ func (ag *Aggregator) Register(name string, rep Reporter) {
 
 }
 
+// HTMLName returns an aggregate name for all the reporters
+func (ag *Aggregator) HTMLName() string {
+	ag.mu.Lock()
+	defer ag.mu.Unlock()
+	result := make([]string, 0, len(ag.reporters))
+	for _, rep := range ag.reporters {
+		result = append(result, rep.HTMLName())
+	}
+	return strings.Join(result, "  +  ")
+}
+
 // Run collects all the health statuses from the default health
 // aggregator.
 func Run(typ topo.TabletType) (map[string]string, error) {
@@ -128,3 +151,8 @@ func Run(typ topo.TabletType) (map[string]string, error) {
 func Register(name string, rep Reporter) {
 	defaultAggregator.Register(name, rep)
 }
+
+// HTMLName returns an aggregate name for the default reporter
+func HTMLName() string {
+	return defaultAggregator.HTMLName()
+}

go/vt/mysqlctl/health.go

@@ -1,27 +1,40 @@
 package mysqlctl
 
 import (
+	"fmt"
+
 	"github.com/youtube/vitess/go/vt/health"
 	"github.com/youtube/vitess/go/vt/topo"
 )
 
+// mySQLReplicationLag implements health.Reporter
+type mysqlReplicationLag struct {
+	mysqld              *Mysqld
+	allowedLagInSeconds int
+}
+
+func (mrl *mysqlReplicationLag) Report(typ topo.TabletType) (status map[string]string, err error) {
+	if !topo.IsSlaveType(typ) {
+		return nil, nil
+	}
+
+	rp, err := mrl.mysqld.SlaveStatus()
+	if err != nil {
+		return nil, err
+	}
+	if int(rp.SecondsBehindMaster) > mrl.allowedLagInSeconds {
+		return map[string]string{health.ReplicationLag: health.ReplicationLagHigh}, nil
+	}
+
+	return nil, nil
+}
+
+func (mrl *mysqlReplicationLag) HTMLName() string {
+	return fmt.Sprintf("MySQLReplicationLag(allowedLag=%v)", mrl.allowedLagInSeconds)
+}
+
 // MySQLReplication lag returns a reporter that reports the MySQL
 // replication lag. It uses the key "replication_lag".
 func MySQLReplicationLag(mysqld *Mysqld, allowedLagInSeconds int) health.Reporter {
-	return health.FunctionReporter(func(typ topo.TabletType) (map[string]string, error) {
-		if !topo.IsSlaveType(typ) {
-			return nil, nil
-		}
-
-		rp, err := mysqld.SlaveStatus()
-		if err != nil {
-			return nil, err
-		}
-		if int(rp.SecondsBehindMaster) > allowedLagInSeconds {
-			return map[string]string{health.ReplicationLag: health.ReplicationLagHigh}, nil
-		}
-
-		return nil, nil
-	})
-
+	return &mysqlReplicationLag{mysqld, allowedLagInSeconds}
 }

go/vt/servenv/status.go

@@ -9,6 +9,7 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"strings"
 	"sync"
 	"time"
 
@@ -40,6 +41,9 @@ func AddStatusFuncs(fmap template.FuncMap) {
 	defer statusMu.Unlock()
 
 	for name, fun := range fmap {
+		if !strings.HasPrefix(name, "github_com_youtube_vitess_") {
+			panic("status func registered without proper prefix, need github_com_youtube_vitess_:" + name)
+		}
 		if _, ok := statusFuncMap[name]; ok {
 			panic("duplicate status func registered: " + name)
 		}
@@ -163,6 +167,11 @@ func statusHandler(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+// StatusURLPath returns the path to the status page.
+func StatusURLPath() string {
+	return "/debug/status"
+}
+
 func init() {
 	var err error
 	hostname, err = os.Hostname()

go/vt/servenv/status_test.go

@@ -13,10 +13,10 @@ import (
 func init() {
 	AddStatusFuncs(
 		template.FuncMap{
-			"to_upper": strings.ToUpper,
+			"github_com_youtube_vitess_to_upper": strings.ToUpper,
 		})
 
-	AddStatusPart("test_part", `{{ to_upper . }}`, func() interface{} {
+	AddStatusPart("test_part", `{{github_com_youtube_vitess_to_upper . }}`, func() interface{} {
 		return "this should be uppercase"
 	})
 	AddStatusSection("test_section", func() string {
@@ -28,7 +28,7 @@ func TestStatus(t *testing.T) {
 	server := httptest.NewServer(nil)
 	defer server.Close()
 
-	resp, err := http.Get(server.URL + "/debug/status")
+	resp, err := http.Get(server.URL + StatusURLPath())
 	if err != nil {
 		t.Fatalf("http.Get: %v", err)
 	}

test/tablet.py

@@ -384,6 +384,9 @@ class Tablet(object):
     config['repl'].update(repl_extra_flags)
     return config
 
+  def get_status(self):
+    return utils.get_status(self.port)
+
   def kill_vttablet(self):
     logging.debug("killing vttablet: %s", self.tablet_alias)
     if self.proc is not None:

test/tabletmanager.py

@@ -507,6 +507,9 @@ class TestTabletManager(unittest.TestCase):
       self.fail('Replication lag parameter not propagated to serving graph: %s' % str(ep))
     self.assertEqual(ep['entries'][0]['health']['replication_lag'], 'high', 'Replication lag parameter not propagated to serving graph: %s' % str(ep))
 
+    # make sure status web page is unhappy
+    self.assertIn('>unhappy</span></div>', tablet_62044.get_status())
+
     # then restart replication, make sure we go back to healthy
     tablet_62044.mquery('', 'start slave')
     timeout = 10
@@ -520,6 +523,9 @@ class TestTabletManager(unittest.TestCase):
       logging.info("Slave tablet replication_lag is gone, good")
       break
 
+    # make sure status web page is healthy
+    self.assertIn('>healthy</span></div>', tablet_62044.get_status())
+
     # kill the tablets
     tablet.kill_tablets([tablet_62344, tablet_62044])