move metrics to service port, add healthz, pprof (#1318)

* combine metrics and healthz

* use root mux and bind pprof routes on debug config opt

Signed-off-by: Evan Baker <rbtr@users.noreply.github.com>

Co-authored-by: Paul Miller <pmiller@microsoft.com>

cns/configuration/configuration.go

@@ -22,7 +22,7 @@ type CNSConfig struct {
 	ChannelMode                 string
 	InitializeFromCNI           bool
 	ManagedSettings             ManagedSettings
-	MetricsBindAddress          string
+	Debug                       bool
 	SyncHostNCTimeoutMs         int
 	SyncHostNCVersionIntervalMs int
 	TLSCertificatePath          string
@@ -150,9 +150,6 @@ func SetCNSConfigDefaults(config *CNSConfig) {
 	if config.ChannelMode == "" {
 		config.ChannelMode = cns.Direct
 	}
-	if config.MetricsBindAddress == "" {
-		config.MetricsBindAddress = ":9090"
-	}
 	if config.SyncHostNCVersionIntervalMs == 0 {
 		config.SyncHostNCVersionIntervalMs = 1000 //nolint:gomnd // default times
 	}

cns/configuration/configuration_test.go

@@ -61,7 +61,6 @@ func TestReadConfigFromFile(t *testing.T) {
 					NodeID:                    "abc",
 					NodeSyncIntervalInSeconds: 30,
 				},
-				MetricsBindAddress:          ":9091",
 				SyncHostNCTimeoutMs:         5,
 				SyncHostNCVersionIntervalMs: 5,
 				TLSCertificatePath:          "/test",
@@ -191,7 +190,7 @@ func TestSetCNSConfigDefaults(t *testing.T) {
 				ManagedSettings: ManagedSettings{
 					NodeSyncIntervalInSeconds: 30,
 				},
-				MetricsBindAddress:          ":9090",
+				Debug:                       false,
 				SyncHostNCTimeoutMs:         500,
 				SyncHostNCVersionIntervalMs: 1000,
 				TelemetrySettings: TelemetrySettings{
@@ -210,7 +209,6 @@ func TestSetCNSConfigDefaults(t *testing.T) {
 				ManagedSettings: ManagedSettings{
 					NodeSyncIntervalInSeconds: 1,
 				},
-				MetricsBindAddress:          ":9091",
 				SyncHostNCTimeoutMs:         5,
 				SyncHostNCVersionIntervalMs: 1,
 				TelemetrySettings: TelemetrySettings{
@@ -226,7 +224,6 @@ func TestSetCNSConfigDefaults(t *testing.T) {
 				ManagedSettings: ManagedSettings{
 					NodeSyncIntervalInSeconds: 1,
 				},
-				MetricsBindAddress:          ":9091",
 				SyncHostNCTimeoutMs:         5,
 				SyncHostNCVersionIntervalMs: 1,
 				TelemetrySettings: TelemetrySettings{

cns/service/main.go

@@ -9,6 +9,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"net/http/pprof"
 	"os"
 	"os/signal"
 	"runtime"
@@ -46,6 +47,7 @@ import (
 	"github.com/Azure/azure-container-networking/store"
 	"github.com/avast/retry-go/v3"
 	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/types"
@@ -53,6 +55,8 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/healthz"
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
 )
 
 const (
@@ -334,8 +338,8 @@ func sendRegisterNodeRequest(
 	httpc *http.Client,
 	httpRestService cns.HTTPService,
 	nodeRegisterRequest cns.NodeRegisterRequest,
-	registerURL string) error {
-
+	registerURL string,
+) error {
 	var body bytes.Buffer
 	err := json.NewEncoder(&body).Encode(nodeRegisterRequest)
 	if err != nil {
@@ -991,10 +995,9 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn
 	})
 
 	manager, err := ctrl.NewManager(kubeConfig, ctrl.Options{
-		Scheme:             nodenetworkconfig.Scheme,
-		MetricsBindAddress: cnsconfig.MetricsBindAddress,
-		Namespace:          "kube-system", // TODO(rbtr): namespace should be in the cns config
-		NewCache:           nodeScopedCache,
+		Scheme:    nodenetworkconfig.Scheme,
+		Namespace: "kube-system", // TODO(rbtr): namespace should be in the cns config
+		NewCache:  nodeScopedCache,
 	})
 	if err != nil {
 		return errors.Wrap(err, "failed to create manager")
@@ -1013,6 +1016,27 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn
 		return errors.Wrapf(err, "failed to setup reconciler with manager")
 	}
 
+	// adding some routes to the root serve mux
+	mux := httpRestServiceImplementation.Listener.GetMux()
+
+	if cnsconfig.Debug {
+		// add pprof endpoints
+		mux.HandleFunc("/debug/pprof/", pprof.Index)
+		mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
+		mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
+		mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
+		mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
+	}
+
+	// add metrics endpoints
+	mux.Handle("/metrics", promhttp.HandlerFor(metrics.Registry, promhttp.HandlerOpts{
+		ErrorHandling: promhttp.HTTPErrorOnError,
+	}))
+
+	// add healthz endpoints
+	healthzhandler := healthz.Handler{}
+	mux.Handle("/healthz", http.StripPrefix("/healthz", &healthzhandler))
+
 	// Start the Manager which starts the reconcile loop.
 	// The Reconciler will send an initial NodeNetworkConfig update to the PoolMonitor, starting the
 	// Monitor's internal loop.
@@ -1057,6 +1081,7 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn
 			}
 		}
 	}()
+
 	logger.Printf("initialized and started SyncHostNCVersion loop")
 
 	return nil