From edda2b14af63a58128ced8a4e677baff376fe95e Mon Sep 17 00:00:00 2001 From: Joakim Karlsson <5434736+roffe@users.noreply.github.com> Date: Thu, 18 Jan 2018 18:35:33 +0100 Subject: [PATCH] Added more metrics and small bugfix + more docs (#291) * fixed pps out using the wrong metrics * adding debug * name fixes adding more metrics * missing mustregister * missing bracer * added more metrics docs * fixed faulty names * fixed more faulty names --- Documentation/metrics.md | 30 ++++++- .../network_services_controller.go | 80 ++++++++++++++++--- 2 files changed, 98 insertions(+), 12 deletions(-) diff --git a/Documentation/metrics.md b/Documentation/metrics.md index d227fa98..47eb9104 100644 --- a/Documentation/metrics.md +++ b/Documentation/metrics.md @@ -35,4 +35,32 @@ For example: k8s-app: kube-router annotations: prometheus.io/scrape: "true" - prometheus.io/port: "8080" \ No newline at end of file + prometheus.io/port: "8080" + +## Avail metrics + +The following metrics is exposed by kube-router prefixed by `kube_router_` + +* service_total_connections + Total connections made to the service since creation +* service_packets_in + Total n/o packets received by service +* service_packets_out + Total n/o packets sent by service +* service_bytes_in + Total bytes received by the service +* service_bytes_out + Total bytes sent by the service +* service_pps_in + Incoming packets per second +* service_pps_out + Outgoing packets per second +* service_cps + Connections per second +* service_bps_in + Incoming bytes per second +* service_bps_out + Outgoing bytes per second + +To get a grouped list of CPS for each service a Prometheus query could look like this e.g: +`sum(kube_router_service_cps) by (namespace, service_name)` \ No newline at end of file diff --git a/app/controllers/network_services_controller.go b/app/controllers/network_services_controller.go index 4defb60d..5a49f850 100644 --- a/app/controllers/network_services_controller.go +++ b/app/controllers/network_services_controller.go @@ -45,11 +45,31 @@ const ( ) var ( - h *ipvs.Handle - serviceActiveConn = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + h *ipvs.Handle + serviceTotalConn = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: namespace, - Name: "service_active_connections", - Help: "Active conntection to service", + Name: "service_total_connections", + Help: "Total conntection to service", + }, []string{"namespace", "service_name", "service_vip"}) + servicePacketsIn = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Name: "service_packets_in", + Help: "Total incoming packets", + }, []string{"namespace", "service_name", "service_vip"}) + servicePacketsOut = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Name: "service_packets_out", + Help: "Total outoging packets", + }, []string{"namespace", "service_name", "service_vip"}) + serviceBytesIn = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Name: "service_bytes_in", + Help: "Total incoming bytes", + }, []string{"namespace", "service_name", "service_vip"}) + serviceBytesOut = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Name: "service_bytes_out", + Help: "Total outoging bytes", }, []string{"namespace", "service_name", "service_vip"}) servicePpsIn = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: namespace, @@ -61,6 +81,21 @@ var ( Name: "service_pps_out", Help: "Outoging packets per second", }, []string{"namespace", "service_name", "service_vip"}) + serviceCPS = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Name: "service_cps", + Help: "Service connections per second", + }, []string{"namespace", "service_name", "service_vip"}) + serviceBpsIn = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Name: "service_bps_in", + Help: "Incoming bytes per second", + }, []string{"namespace", "service_name", "service_vip"}) + serviceBpsOut = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Name: "service_bps_out", + Help: "Outoging bytes per second", + }, []string{"namespace", "service_name", "service_vip"}) ) // NetworkServicesController enables local node as network service proxy through IPVS/LVS. @@ -132,9 +167,17 @@ func (nsc *NetworkServicesController) Run(stopCh <-chan struct{}, wg *sync.WaitG } // register metrics - prometheus.MustRegister(serviceActiveConn) + prometheus.MustRegister(serviceBpsIn) + prometheus.MustRegister(serviceBpsOut) + prometheus.MustRegister(serviceBytesIn) + prometheus.MustRegister(serviceBytesOut) + prometheus.MustRegister(serviceCPS) + prometheus.MustRegister(servicePacketsIn) + prometheus.MustRegister(servicePacketsOut) prometheus.MustRegister(servicePpsIn) prometheus.MustRegister(servicePpsOut) + prometheus.MustRegister(serviceTotalConn) + http.Handle(nsc.MetricsPath, promhttp.Handler()) go http.ListenAndServe(":"+strconv.Itoa(nsc.MetricsPort), nil) @@ -748,16 +791,31 @@ func (nsc *NetworkServicesController) publishMetrics(serviceInfoMap serviceInfoM if strings.Compare(svc.clusterIP.String(), ipvsSvc.Address.String()) == 0 && protocol == ipvsSvc.Protocol && uint16(svc.port) == ipvsSvc.Port { glog.Infof("Publishing prometheus metrics " + svc.clusterIP.String() + ":" + strconv.Itoa(svc.port)) - serviceActiveConn.WithLabelValues(svc.namespace, svc.name, svc.clusterIP.String()).Set(float64(ipvsSvc.Stats.Connections)) - servicePpsIn.WithLabelValues(svc.namespace, svc.name, svc.clusterIP.String()).Set(float64(ipvsSvc.Stats.PacketsIn)) - servicePpsOut.WithLabelValues(svc.namespace, svc.name, svc.clusterIP.String()).Set(float64(ipvsSvc.Stats.PacketsIn)) + serviceBpsIn.WithLabelValues(svc.namespace, svc.name, svc.clusterIP.String()).Set(float64(ipvsSvc.Stats.BPSIn)) + serviceBpsOut.WithLabelValues(svc.namespace, svc.name, svc.clusterIP.String()).Set(float64(ipvsSvc.Stats.BPSOut)) + serviceBytesIn.WithLabelValues(svc.namespace, svc.name, svc.clusterIP.String()).Set(float64(ipvsSvc.Stats.BytesIn)) + serviceBytesOut.WithLabelValues(svc.namespace, svc.name, svc.clusterIP.String()).Set(float64(ipvsSvc.Stats.BytesOut)) + serviceCPS.WithLabelValues(svc.namespace, svc.name, svc.clusterIP.String()).Set(float64(ipvsSvc.Stats.CPS)) + servicePacketsIn.WithLabelValues(svc.namespace, svc.name, svc.clusterIP.String()).Set(float64(ipvsSvc.Stats.PacketsIn)) + servicePacketsOut.WithLabelValues(svc.namespace, svc.name, svc.clusterIP.String()).Set(float64(ipvsSvc.Stats.PacketsOut)) + servicePpsIn.WithLabelValues(svc.namespace, svc.name, svc.clusterIP.String()).Set(float64(ipvsSvc.Stats.PPSIn)) + servicePpsOut.WithLabelValues(svc.namespace, svc.name, svc.clusterIP.String()).Set(float64(ipvsSvc.Stats.PPSOut)) + serviceTotalConn.WithLabelValues(svc.namespace, svc.name, svc.clusterIP.String()).Set(float64(ipvsSvc.Stats.Connections)) + } if strings.Compare(nsc.nodeIP.String(), ipvsSvc.Address.String()) == 0 && protocol == ipvsSvc.Protocol && uint16(svc.port) == ipvsSvc.Port { glog.Infof("Publishing prometheus metrics " + nsc.nodeIP.String() + ":" + strconv.Itoa(svc.port)) - serviceActiveConn.WithLabelValues(svc.namespace, svc.name, nsc.nodeIP.String()).Set(float64(ipvsSvc.Stats.Connections)) - servicePpsIn.WithLabelValues(svc.namespace, svc.name, nsc.nodeIP.String()).Set(float64(ipvsSvc.Stats.PacketsIn)) - servicePpsOut.WithLabelValues(svc.namespace, svc.name, nsc.nodeIP.String()).Set(float64(ipvsSvc.Stats.PacketsIn)) + serviceBpsIn.WithLabelValues(svc.namespace, svc.name, nsc.nodeIP.String()).Set(float64(ipvsSvc.Stats.BPSIn)) + serviceBpsOut.WithLabelValues(svc.namespace, svc.name, nsc.nodeIP.String()).Set(float64(ipvsSvc.Stats.BPSOut)) + serviceBytesIn.WithLabelValues(svc.namespace, svc.name, nsc.nodeIP.String()).Set(float64(ipvsSvc.Stats.BytesIn)) + serviceBytesOut.WithLabelValues(svc.namespace, svc.name, nsc.nodeIP.String()).Set(float64(ipvsSvc.Stats.BytesOut)) + serviceCPS.WithLabelValues(svc.namespace, svc.name, nsc.nodeIP.String()).Set(float64(ipvsSvc.Stats.CPS)) + servicePacketsIn.WithLabelValues(svc.namespace, svc.name, nsc.nodeIP.String()).Set(float64(ipvsSvc.Stats.PacketsIn)) + servicePacketsOut.WithLabelValues(svc.namespace, svc.name, nsc.nodeIP.String()).Set(float64(ipvsSvc.Stats.PacketsOut)) + servicePpsIn.WithLabelValues(svc.namespace, svc.name, nsc.nodeIP.String()).Set(float64(ipvsSvc.Stats.PPSIn)) + servicePpsOut.WithLabelValues(svc.namespace, svc.name, nsc.nodeIP.String()).Set(float64(ipvsSvc.Stats.PPSOut)) + serviceTotalConn.WithLabelValues(svc.namespace, svc.name, nsc.nodeIP.String()).Set(float64(ipvsSvc.Stats.Connections)) } } }