move health, metrics to sepearate packages (#404)

This commit is contained in:
Murali Reddy 2018-04-22 11:36:47 +00:00 committed by GitHub
parent 6eece2d737
commit 1a0bfa2dfb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 84 additions and 74 deletions

View File

@ -10,6 +10,8 @@ import (
"syscall"
"github.com/cloudnativelabs/kube-router/pkg/controllers"
"github.com/cloudnativelabs/kube-router/pkg/healthcheck"
"github.com/cloudnativelabs/kube-router/pkg/metrics"
"github.com/cloudnativelabs/kube-router/pkg/options"
"github.com/golang/glog"
@ -73,7 +75,7 @@ func (kr *KubeRouter) Run() error {
var err error
var wg sync.WaitGroup
healthChan := make(chan *controllers.ControllerHeartbeat, 10)
healthChan := make(chan *healthcheck.ControllerHeartbeat, 10)
defer close(healthChan)
stopCh := make(chan struct{})
@ -83,7 +85,7 @@ func (kr *KubeRouter) Run() error {
os.Exit(0)
}
hc, err := controllers.NewHealthController(kr.Config)
hc, err := healthcheck.NewHealthController(kr.Config)
if err != nil {
return errors.New("Failed to create health controller: " + err.Error())
}
@ -92,7 +94,7 @@ func (kr *KubeRouter) Run() error {
if (kr.Config.MetricsPort > 0) && (kr.Config.MetricsPort <= 65535) {
kr.Config.MetricsEnabled = true
mc, err := controllers.NewMetricsController(kr.Client, kr.Config)
mc, err := metrics.NewMetricsController(kr.Client, kr.Config)
if err != nil {
return errors.New("Failed to create metrics controller: " + err.Error())
}

View File

@ -11,6 +11,8 @@ import (
"sync"
"time"
"github.com/cloudnativelabs/kube-router/pkg/healthcheck"
"github.com/cloudnativelabs/kube-router/pkg/metrics"
"github.com/cloudnativelabs/kube-router/pkg/options"
"github.com/cloudnativelabs/kube-router/pkg/utils"
"github.com/coreos/go-iptables/iptables"
@ -115,7 +117,7 @@ type protocolAndPort struct {
}
// Run runs forver till we receive notification on stopCh
func (npc *NetworkPolicyController) Run(healthChan chan<- *ControllerHeartbeat, stopCh <-chan struct{}, wg *sync.WaitGroup) {
func (npc *NetworkPolicyController) Run(healthChan chan<- *healthcheck.ControllerHeartbeat, stopCh <-chan struct{}, wg *sync.WaitGroup) {
t := time.NewTicker(npc.syncPeriod)
defer t.Stop()
defer wg.Done()
@ -136,7 +138,7 @@ func (npc *NetworkPolicyController) Run(healthChan chan<- *ControllerHeartbeat,
if err != nil {
glog.Errorf("Error during periodic sync: " + err.Error())
} else {
sendHeartBeat(healthChan, "NPC")
healthcheck.SendHeartBeat(healthChan, "NPC")
}
npc.readyForUpdates = true
select {
@ -206,7 +208,7 @@ func (npc *NetworkPolicyController) Sync() error {
defer func() {
endTime := time.Since(start)
if npc.MetricsEnabled {
controllerIptablesSyncTime.WithLabelValues().Set(float64(endTime))
metrics.ControllerIptablesSyncTime.WithLabelValues().Set(float64(endTime))
}
glog.V(1).Infof("sync iptables took %v", endTime)
}()
@ -1480,7 +1482,7 @@ func NewNetworkPolicyController(clientset kubernetes.Interface,
if config.MetricsEnabled {
//Register the metrics for this controller
prometheus.MustRegister(controllerIptablesSyncTime)
prometheus.MustRegister(metrics.ControllerIptablesSyncTime)
npc.MetricsEnabled = true
}

View File

@ -19,6 +19,8 @@ import (
"github.com/aws/aws-sdk-go/aws/ec2metadata"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/cloudnativelabs/kube-router/pkg/healthcheck"
"github.com/cloudnativelabs/kube-router/pkg/metrics"
"github.com/cloudnativelabs/kube-router/pkg/options"
"github.com/cloudnativelabs/kube-router/pkg/utils"
"github.com/coreos/go-iptables/iptables"
@ -107,7 +109,7 @@ type NetworkRoutingController struct {
}
// Run runs forever until we are notified on stop channel
func (nrc *NetworkRoutingController) Run(healthChan chan<- *ControllerHeartbeat, stopCh <-chan struct{}, wg *sync.WaitGroup) {
func (nrc *NetworkRoutingController) Run(healthChan chan<- *healthcheck.ControllerHeartbeat, stopCh <-chan struct{}, wg *sync.WaitGroup) {
cidr, err := utils.GetPodCidrFromCniSpec(nrc.cniConfFile)
if err != nil {
glog.Errorf("Failed to get pod CIDR from CNI conf file: %s", err.Error())
@ -280,7 +282,7 @@ func (nrc *NetworkRoutingController) Run(healthChan chan<- *ControllerHeartbeat,
nrc.syncInternalPeers()
}
sendHeartBeat(healthChan, "NRC")
healthcheck.SendHeartBeat(healthChan, "NRC")
select {
case <-stopCh:
@ -367,7 +369,7 @@ func (nrc *NetworkRoutingController) watchBgpUpdates() {
case *gobgp.WatchEventBestPath:
glog.V(3).Info("Processing bgp route advertisement from peer")
if nrc.MetricsEnabled {
controllerBGPadvertisementsReceived.WithLabelValues().Add(float64(1))
metrics.ControllerBGPadvertisementsReceived.WithLabelValues().Add(float64(1))
}
for _, path := range msg.PathList {
if path.IsLocal() {
@ -1141,7 +1143,7 @@ func (nrc *NetworkRoutingController) syncInternalPeers() {
start := time.Now()
defer func() {
endTime := time.Since(start)
controllerBGPInternalPeersSyncTime.WithLabelValues().Set(float64(endTime))
metrics.ControllerBGPInternalPeersSyncTime.WithLabelValues().Set(float64(endTime))
glog.V(2).Infof("Syncing BGP peers for the node took %v", endTime)
}()
@ -1152,7 +1154,7 @@ func (nrc *NetworkRoutingController) syncInternalPeers() {
return
}
controllerBPGpeers.WithLabelValues().Set(float64(len(nodes.Items)))
metrics.ControllerBPGpeers.WithLabelValues().Set(float64(len(nodes.Items)))
// establish peer and add Pod CIDRs with current set of nodes
currentNodes := make([]string, 0)
for _, node := range nodes.Items {
@ -1782,9 +1784,9 @@ func NewNetworkRoutingController(clientset kubernetes.Interface,
nrc := NetworkRoutingController{}
if kubeRouterConfig.MetricsEnabled {
//Register the metrics for this controller
prometheus.MustRegister(controllerBGPadvertisementsReceived)
prometheus.MustRegister(controllerBGPInternalPeersSyncTime)
prometheus.MustRegister(controllerBPGpeers)
prometheus.MustRegister(metrics.ControllerBGPadvertisementsReceived)
prometheus.MustRegister(metrics.ControllerBGPInternalPeersSyncTime)
prometheus.MustRegister(metrics.ControllerBPGpeers)
nrc.MetricsEnabled = true
}

View File

@ -18,6 +18,8 @@ import (
"syscall"
"time"
"github.com/cloudnativelabs/kube-router/pkg/healthcheck"
"github.com/cloudnativelabs/kube-router/pkg/metrics"
"github.com/cloudnativelabs/kube-router/pkg/options"
"github.com/cloudnativelabs/kube-router/pkg/utils"
"github.com/coreos/go-iptables/iptables"
@ -42,7 +44,6 @@ const (
IFACE_HAS_ADDR = "file exists"
IFACE_HAS_NO_ADDR = "cannot assign requested address"
IPVS_SERVER_EXISTS = "file exists"
namespace = "kube_router"
svcDSRAnnotation = "kube-router.io/service.dsr"
svcSchedulerAnnotation = "kube-router.io/service.scheduler"
@ -211,7 +212,7 @@ type endpointsInfo struct {
type endpointsInfoMap map[string][]endpointsInfo
// Run periodically sync ipvs configuration to reflect desired state of services and endpoints
func (nsc *NetworkServicesController) Run(healthChan chan<- *ControllerHeartbeat, stopCh <-chan struct{}, wg *sync.WaitGroup) error {
func (nsc *NetworkServicesController) Run(healthChan chan<- *healthcheck.ControllerHeartbeat, stopCh <-chan struct{}, wg *sync.WaitGroup) error {
t := time.NewTicker(nsc.syncPeriod)
defer t.Stop()
@ -245,7 +246,7 @@ func (nsc *NetworkServicesController) Run(healthChan chan<- *ControllerHeartbeat
if err != nil {
glog.Errorf("Error during periodic ipvs sync: " + err.Error())
} else {
sendHeartBeat(healthChan, "NSC")
healthcheck.SendHeartBeat(healthChan, "NSC")
}
nsc.readyForUpdates = true
select {
@ -286,7 +287,7 @@ func (nsc *NetworkServicesController) publishMetrics(serviceInfoMap serviceInfoM
defer func() {
endTime := time.Since(start)
glog.V(2).Infof("Publishing IPVS metrics took %v", endTime)
controllerIpvsMetricsExportTime.WithLabelValues().Set(float64(endTime))
metrics.ControllerIpvsMetricsExportTime.WithLabelValues().Set(float64(endTime))
}()
ipvsSvcs, err := nsc.ln.ipvsGetServices()
@ -332,17 +333,17 @@ func (nsc *NetworkServicesController) publishMetrics(serviceInfoMap serviceInfoM
if pushMetric {
glog.V(3).Infof("Publishing metrics for %s/%s (%s:%d/%s)", svc.namespace, svc.name, svcVip, svc.port, svc.protocol)
serviceBpsIn.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.BPSIn))
serviceBpsOut.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.BPSOut))
serviceBytesIn.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.BytesIn))
serviceBytesOut.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.BytesOut))
serviceCPS.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.CPS))
servicePacketsIn.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.PacketsIn))
servicePacketsOut.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.PacketsOut))
servicePpsIn.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.PPSIn))
servicePpsOut.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.PPSOut))
serviceTotalConn.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.Connections))
controllerIpvsServices.WithLabelValues().Set(float64(len(ipvsSvcs)))
metrics.ServiceBpsIn.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.BPSIn))
metrics.ServiceBpsOut.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.BPSOut))
metrics.ServiceBytesIn.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.BytesIn))
metrics.ServiceBytesOut.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.BytesOut))
metrics.ServiceCPS.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.CPS))
metrics.ServicePacketsIn.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.PacketsIn))
metrics.ServicePacketsOut.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.PacketsOut))
metrics.ServicePpsIn.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.PPSIn))
metrics.ServicePpsOut.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.PPSOut))
metrics.ServiceTotalConn.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.Connections))
metrics.ControllerIpvsServices.WithLabelValues().Set(float64(len(ipvsSvcs)))
}
}
}
@ -432,7 +433,7 @@ func (nsc *NetworkServicesController) syncIpvsServices(serviceInfoMap serviceInf
defer func() {
endTime := time.Since(start)
if nsc.MetricsEnabled {
controllerIpvsServicesSyncTime.WithLabelValues().Set(float64(endTime))
metrics.ControllerIpvsServicesSyncTime.WithLabelValues().Set(float64(endTime))
}
glog.V(1).Infof("sync ipvs services took %v", endTime)
}()
@ -1798,18 +1799,18 @@ func NewNetworkServicesController(clientset kubernetes.Interface,
if config.MetricsEnabled {
//Register the metrics for this controller
prometheus.MustRegister(controllerIpvsServices)
prometheus.MustRegister(controllerIpvsServicesSyncTime)
prometheus.MustRegister(serviceBpsIn)
prometheus.MustRegister(serviceBpsOut)
prometheus.MustRegister(serviceBytesIn)
prometheus.MustRegister(serviceBytesOut)
prometheus.MustRegister(serviceCPS)
prometheus.MustRegister(servicePacketsIn)
prometheus.MustRegister(servicePacketsOut)
prometheus.MustRegister(servicePpsIn)
prometheus.MustRegister(servicePpsOut)
prometheus.MustRegister(serviceTotalConn)
prometheus.MustRegister(metrics.ControllerIpvsServices)
prometheus.MustRegister(metrics.ControllerIpvsServicesSyncTime)
prometheus.MustRegister(metrics.ServiceBpsIn)
prometheus.MustRegister(metrics.ServiceBpsOut)
prometheus.MustRegister(metrics.ServiceBytesIn)
prometheus.MustRegister(metrics.ServiceBytesOut)
prometheus.MustRegister(metrics.ServiceCPS)
prometheus.MustRegister(metrics.ServicePacketsIn)
prometheus.MustRegister(metrics.ServicePacketsOut)
prometheus.MustRegister(metrics.ServicePpsIn)
prometheus.MustRegister(metrics.ServicePpsOut)
prometheus.MustRegister(metrics.ServiceTotalConn)
nsc.MetricsEnabled = true
}

View File

@ -1,4 +1,4 @@
package controllers
package healthcheck
import (
"net/http"
@ -35,8 +35,8 @@ type HealthStats struct {
NetworkServicesControllerAlive time.Time
}
//sendHeartBeat sends a heartbeat on the passed channel
func sendHeartBeat(channel chan<- *ControllerHeartbeat, controller string) {
//SendHeartBeat sends a heartbeat on the passed channel
func SendHeartBeat(channel chan<- *ControllerHeartbeat, controller string) {
heartbeat := ControllerHeartbeat{
Component: controller,
LastHeartBeat: time.Now(),

View File

@ -1,4 +1,4 @@
package controllers
package metrics
import (
"net"
@ -7,6 +7,7 @@ import (
"sync"
"time"
"github.com/cloudnativelabs/kube-router/pkg/healthcheck"
"github.com/cloudnativelabs/kube-router/pkg/options"
"github.com/golang/glog"
"github.com/prometheus/client_golang/prometheus"
@ -15,93 +16,97 @@ import (
"k8s.io/client-go/kubernetes"
)
const (
namespace = "kube_router"
)
var (
serviceTotalConn = prometheus.NewGaugeVec(prometheus.GaugeOpts{
ServiceTotalConn = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Name: "service_total_connections",
Help: "Total incoming connections made",
}, []string{"namespace", "service_name", "service_vip", "protocol", "port"})
servicePacketsIn = prometheus.NewGaugeVec(prometheus.GaugeOpts{
ServicePacketsIn = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Name: "service_packets_in",
Help: "Total incoming packets",
}, []string{"namespace", "service_name", "service_vip", "protocol", "port"})
servicePacketsOut = prometheus.NewGaugeVec(prometheus.GaugeOpts{
ServicePacketsOut = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Name: "service_packets_out",
Help: "Total outgoing packets",
}, []string{"namespace", "service_name", "service_vip", "protocol", "port"})
serviceBytesIn = prometheus.NewGaugeVec(prometheus.GaugeOpts{
ServiceBytesIn = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Name: "service_bytes_in",
Help: "Total incoming bytes",
}, []string{"namespace", "service_name", "service_vip", "protocol", "port"})
serviceBytesOut = prometheus.NewGaugeVec(prometheus.GaugeOpts{
ServiceBytesOut = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Name: "service_bytes_out",
Help: "Total outgoing bytes",
}, []string{"namespace", "service_name", "service_vip", "protocol", "port"})
servicePpsIn = prometheus.NewGaugeVec(prometheus.GaugeOpts{
ServicePpsIn = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Name: "service_pps_in",
Help: "Incoming packets per second",
}, []string{"namespace", "service_name", "service_vip", "protocol", "port"})
servicePpsOut = prometheus.NewGaugeVec(prometheus.GaugeOpts{
ServicePpsOut = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Name: "service_pps_out",
Help: "Outgoing packets per second",
}, []string{"namespace", "service_name", "service_vip", "protocol", "port"})
serviceCPS = prometheus.NewGaugeVec(prometheus.GaugeOpts{
ServiceCPS = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Name: "service_cps",
Help: "Service connections per second",
}, []string{"namespace", "service_name", "service_vip", "protocol", "port"})
serviceBpsIn = prometheus.NewGaugeVec(prometheus.GaugeOpts{
ServiceBpsIn = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Name: "service_bps_in",
Help: "Incoming bytes per second",
}, []string{"namespace", "service_name", "service_vip", "protocol", "port"})
serviceBpsOut = prometheus.NewGaugeVec(prometheus.GaugeOpts{
ServiceBpsOut = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Name: "service_bps_out",
Help: "Outgoing bytes per second",
}, []string{"namespace", "service_name", "service_vip", "protocol", "port"})
controllerIpvsServices = prometheus.NewGaugeVec(prometheus.GaugeOpts{
ControllerIpvsServices = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Name: "controller_ipvs_services",
Help: "Number of ipvs services in the instance",
}, []string{})
controllerIptablesSyncTime = prometheus.NewGaugeVec(prometheus.GaugeOpts{
ControllerIptablesSyncTime = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Name: "controller_iptables_sync_time",
Help: "Time it took for controller to sync iptables",
}, []string{})
controllerPublishMetricsTime = prometheus.NewGaugeVec(prometheus.GaugeOpts{
ControllerPublishMetricsTime = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Name: "controller_publish_metrics_time",
Help: "Time it took to publish metrics",
}, []string{})
controllerIpvsServicesSyncTime = prometheus.NewGaugeVec(prometheus.GaugeOpts{
ControllerIpvsServicesSyncTime = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Name: "controller_ipvs_services_sync_time",
Help: "Time it took for controller to sync ipvs services",
}, []string{})
controllerBPGpeers = prometheus.NewGaugeVec(prometheus.GaugeOpts{
ControllerBPGpeers = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Name: "controller_bgp_peers",
Help: "BGP peers in the runtime configuration",
}, []string{})
controllerBGPInternalPeersSyncTime = prometheus.NewGaugeVec(prometheus.GaugeOpts{
ControllerBGPInternalPeersSyncTime = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Name: "controller_bgp_internal_peers_sync_time",
Help: "Time it took to sync internal bgp peers",
}, []string{})
controllerBGPadvertisementsReceived = prometheus.NewGaugeVec(prometheus.GaugeOpts{
ControllerBGPadvertisementsReceived = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Name: "controller_bgp_advertisements_received",
Help: "Time it took to sync internal bgp peers",
}, []string{})
controllerIpvsMetricsExportTime = prometheus.NewGaugeVec(prometheus.GaugeOpts{
ControllerIpvsMetricsExportTime = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Name: "controller_ipvs_metrics_export_time",
Help: "Time it took to export metrics",
@ -110,22 +115,20 @@ var (
// MetricsController Holds settings for the metrics controller
type MetricsController struct {
endpointsMap endpointsInfoMap
MetricsPath string
MetricsPort uint16
mu sync.Mutex
nodeIP net.IP
serviceMap serviceInfoMap
MetricsPath string
MetricsPort uint16
mu sync.Mutex
nodeIP net.IP
}
// Run prometheus metrics controller
func (mc *MetricsController) Run(healthChan chan<- *ControllerHeartbeat, stopCh <-chan struct{}, wg *sync.WaitGroup) error {
func (mc *MetricsController) Run(healthChan chan<- *healthcheck.ControllerHeartbeat, stopCh <-chan struct{}, wg *sync.WaitGroup) error {
t := time.NewTicker(3 * time.Second)
defer wg.Done()
glog.Info("Starting metrics controller")
// register metrics for this controller
prometheus.MustRegister(controllerIpvsMetricsExportTime)
prometheus.MustRegister(ControllerIpvsMetricsExportTime)
srv := &http.Server{Addr: ":" + strconv.Itoa(int(mc.MetricsPort)), Handler: http.DefaultServeMux}
@ -139,7 +142,7 @@ func (mc *MetricsController) Run(healthChan chan<- *ControllerHeartbeat, stopCh
}
}()
for {
sendHeartBeat(healthChan, "MC")
healthcheck.SendHeartBeat(healthChan, "MC")
select {
case <-stopCh:
glog.Infof("Shutting down metrics controller")