diff --git a/pkg/controllers/proxy/network_services_controller.go b/pkg/controllers/proxy/network_services_controller.go index b7d163bf..f3af4baf 100644 --- a/pkg/controllers/proxy/network_services_controller.go +++ b/pkg/controllers/proxy/network_services_controller.go @@ -22,7 +22,6 @@ import ( "github.com/cloudnativelabs/kube-router/v2/pkg/utils" "github.com/coreos/go-iptables/iptables" "github.com/moby/ipvs" - "github.com/prometheus/client_golang/prometheus" "github.com/vishvananda/netlink" v1 "k8s.io/api/core/v1" discovery "k8s.io/api/discovery/v1" @@ -720,180 +719,6 @@ func (nsc *NetworkServicesController) syncIpvsFirewall() error { return nil } -func (*NetworkServicesController) Describe(ch chan<- *prometheus.Desc) { - ch <- metrics.ServiceBpsIn - ch <- metrics.ServiceBpsOut - ch <- metrics.ServiceBytesIn - ch <- metrics.ServiceBytesOut - ch <- metrics.ServiceCPS - ch <- metrics.ServicePacketsIn - ch <- metrics.ServicePacketsOut - ch <- metrics.ServicePpsIn - ch <- metrics.ServicePpsOut - ch <- metrics.ServiceTotalConn - ch <- metrics.ControllerIpvsServices -} - -func (nsc *NetworkServicesController) Collect(ch chan<- prometheus.Metric) { - start := time.Now() - defer func() { - endTime := time.Since(start) - klog.V(2).Infof("Publishing IPVS metrics took %v", endTime) - if nsc.MetricsEnabled { - metrics.ControllerIpvsMetricsExportTime.Observe(endTime.Seconds()) - } - }() - - ipvsHandle, err := ipvs.New("") - if err != nil { - klog.Errorf("failed to initialize ipvs handle: %v", err) - return - } - defer ipvsHandle.Close() - - ipvsSvcs, err := ipvsHandle.GetServices() - if err != nil { - klog.Errorf("failed to list IPVS services: %v", err) - return - } - - type svcMapKey struct { - ip string - uPort uint16 - protocol uint16 - } - - serviceMap := map[svcMapKey]*serviceInfo{} - for _, svc := range nsc.getServiceMap() { - key := svcMapKey{} - key.uPort, err = safecast.ToUint16(svc.port) - if err != nil { - klog.Errorf("failed to convert port %d to uint16: %v", svc.port, err) - continue - } - key.protocol = convertSvcProtoToSysCallProto(svc.protocol) - - for _, ip := range svc.clusterIPs { - key.ip = ip - serviceMap[key] = svc - } - for _, ip := range svc.externalIPs { - key.ip = ip - serviceMap[key] = svc - } - if svc.nodePort != 0 { - key.ip = nsc.krNode.GetPrimaryNodeIP().String() - key.uPort, err = safecast.ToUint16(svc.nodePort) - if err != nil { - klog.Errorf("failed to convert nodePort %d to uint16: %v", svc.nodePort, err) - continue - } - serviceMap[key] = svc - } - } - - klog.V(1).Info("Publishing IPVS metrics") - for _, ipvsSvc := range ipvsSvcs { - key := svcMapKey{ - ip: ipvsSvc.Address.String(), - uPort: ipvsSvc.Port, - protocol: ipvsSvc.Protocol, - } - - svc, ok := serviceMap[key] - if !ok { - continue - } - - klog.V(3).Infof("Publishing metrics for %s/%s (%s:%d/%s)", - svc.namespace, svc.name, key.ip, key.uPort, svc.protocol) - - labelValues := []string{ - svc.namespace, - svc.name, - key.ip, - svc.protocol, - strconv.Itoa(int(key.uPort)), - } - - ch <- prometheus.MustNewConstMetric( - metrics.ServiceBpsIn, - prometheus.GaugeValue, - float64(ipvsSvc.Stats.BPSIn), - labelValues..., - ) - - ch <- prometheus.MustNewConstMetric( - metrics.ServiceBpsOut, - prometheus.GaugeValue, - float64(ipvsSvc.Stats.BPSOut), - labelValues..., - ) - - ch <- prometheus.MustNewConstMetric( - metrics.ServiceBytesIn, - prometheus.CounterValue, - float64(ipvsSvc.Stats.BytesIn), - labelValues..., - ) - - ch <- prometheus.MustNewConstMetric( - metrics.ServiceBytesOut, - prometheus.CounterValue, - float64(ipvsSvc.Stats.BytesOut), - labelValues..., - ) - - ch <- prometheus.MustNewConstMetric( - metrics.ServiceCPS, - prometheus.GaugeValue, - float64(ipvsSvc.Stats.CPS), - labelValues..., - ) - - ch <- prometheus.MustNewConstMetric( - metrics.ServicePacketsIn, - prometheus.CounterValue, - float64(ipvsSvc.Stats.PacketsIn), - labelValues..., - ) - - ch <- prometheus.MustNewConstMetric( - metrics.ServicePacketsOut, - prometheus.CounterValue, - float64(ipvsSvc.Stats.PacketsOut), - labelValues..., - ) - - ch <- prometheus.MustNewConstMetric( - metrics.ServicePpsIn, - prometheus.GaugeValue, - float64(ipvsSvc.Stats.PPSIn), - labelValues..., - ) - - ch <- prometheus.MustNewConstMetric( - metrics.ServicePpsOut, - prometheus.GaugeValue, - float64(ipvsSvc.Stats.PPSOut), - labelValues..., - ) - - ch <- prometheus.MustNewConstMetric( - metrics.ServiceTotalConn, - prometheus.CounterValue, - float64(ipvsSvc.Stats.Connections), - labelValues..., - ) - } - - ch <- prometheus.MustNewConstMetric( - metrics.ControllerIpvsServices, - prometheus.GaugeValue, - float64(len(ipvsSvcs)), - ) -} - // OnEndpointsUpdate handle change in endpoints update from the API server func (nsc *NetworkServicesController) OnEndpointsUpdate(es *discovery.EndpointSlice) { diff --git a/pkg/controllers/proxy/network_services_metrics.go b/pkg/controllers/proxy/network_services_metrics.go new file mode 100644 index 00000000..805753ab --- /dev/null +++ b/pkg/controllers/proxy/network_services_metrics.go @@ -0,0 +1,186 @@ +package proxy + +import ( + "strconv" + "time" + + "github.com/ccoveille/go-safecast" + "github.com/cloudnativelabs/kube-router/v2/pkg/metrics" + "github.com/moby/ipvs" + "github.com/prometheus/client_golang/prometheus" + "k8s.io/klog/v2" +) + +func (*NetworkServicesController) Describe(ch chan<- *prometheus.Desc) { + ch <- metrics.ServiceBpsIn + ch <- metrics.ServiceBpsOut + ch <- metrics.ServiceBytesIn + ch <- metrics.ServiceBytesOut + ch <- metrics.ServiceCPS + ch <- metrics.ServicePacketsIn + ch <- metrics.ServicePacketsOut + ch <- metrics.ServicePpsIn + ch <- metrics.ServicePpsOut + ch <- metrics.ServiceTotalConn + ch <- metrics.ControllerIpvsServices +} + +func (nsc *NetworkServicesController) Collect(ch chan<- prometheus.Metric) { + start := time.Now() + defer func() { + endTime := time.Since(start) + klog.V(2).Infof("Publishing IPVS metrics took %v", endTime) + if nsc.MetricsEnabled { + metrics.ControllerIpvsMetricsExportTime.Observe(endTime.Seconds()) + } + }() + + ipvsHandle, err := ipvs.New("") + if err != nil { + klog.Errorf("failed to initialize ipvs handle: %v", err) + return + } + defer ipvsHandle.Close() + + ipvsSvcs, err := ipvsHandle.GetServices() + if err != nil { + klog.Errorf("failed to list IPVS services: %v", err) + return + } + + type svcMapKey struct { + ip string + uPort uint16 + protocol uint16 + } + + serviceMap := map[svcMapKey]*serviceInfo{} + for _, svc := range nsc.getServiceMap() { + key := svcMapKey{} + key.uPort, err = safecast.ToUint16(svc.port) + if err != nil { + klog.Errorf("failed to convert port %d to uint16: %v", svc.port, err) + continue + } + key.protocol = convertSvcProtoToSysCallProto(svc.protocol) + + for _, ip := range svc.clusterIPs { + key.ip = ip + serviceMap[key] = svc + } + for _, ip := range svc.externalIPs { + key.ip = ip + serviceMap[key] = svc + } + if svc.nodePort != 0 { + key.ip = nsc.krNode.GetPrimaryNodeIP().String() + key.uPort, err = safecast.ToUint16(svc.nodePort) + if err != nil { + klog.Errorf("failed to convert nodePort %d to uint16: %v", svc.nodePort, err) + continue + } + serviceMap[key] = svc + } + } + + klog.V(1).Info("Publishing IPVS metrics") + for _, ipvsSvc := range ipvsSvcs { + key := svcMapKey{ + ip: ipvsSvc.Address.String(), + uPort: ipvsSvc.Port, + protocol: ipvsSvc.Protocol, + } + + svc, ok := serviceMap[key] + if !ok { + continue + } + + klog.V(3).Infof("Publishing metrics for %s/%s (%s:%d/%s)", + svc.namespace, svc.name, key.ip, key.uPort, svc.protocol) + + labelValues := []string{ + svc.namespace, + svc.name, + key.ip, + svc.protocol, + strconv.Itoa(int(key.uPort)), + } + + ch <- prometheus.MustNewConstMetric( + metrics.ServiceBpsIn, + prometheus.GaugeValue, + float64(ipvsSvc.Stats.BPSIn), + labelValues..., + ) + + ch <- prometheus.MustNewConstMetric( + metrics.ServiceBpsOut, + prometheus.GaugeValue, + float64(ipvsSvc.Stats.BPSOut), + labelValues..., + ) + + ch <- prometheus.MustNewConstMetric( + metrics.ServiceBytesIn, + prometheus.CounterValue, + float64(ipvsSvc.Stats.BytesIn), + labelValues..., + ) + + ch <- prometheus.MustNewConstMetric( + metrics.ServiceBytesOut, + prometheus.CounterValue, + float64(ipvsSvc.Stats.BytesOut), + labelValues..., + ) + + ch <- prometheus.MustNewConstMetric( + metrics.ServiceCPS, + prometheus.GaugeValue, + float64(ipvsSvc.Stats.CPS), + labelValues..., + ) + + ch <- prometheus.MustNewConstMetric( + metrics.ServicePacketsIn, + prometheus.CounterValue, + float64(ipvsSvc.Stats.PacketsIn), + labelValues..., + ) + + ch <- prometheus.MustNewConstMetric( + metrics.ServicePacketsOut, + prometheus.CounterValue, + float64(ipvsSvc.Stats.PacketsOut), + labelValues..., + ) + + ch <- prometheus.MustNewConstMetric( + metrics.ServicePpsIn, + prometheus.GaugeValue, + float64(ipvsSvc.Stats.PPSIn), + labelValues..., + ) + + ch <- prometheus.MustNewConstMetric( + metrics.ServicePpsOut, + prometheus.GaugeValue, + float64(ipvsSvc.Stats.PPSOut), + labelValues..., + ) + + ch <- prometheus.MustNewConstMetric( + metrics.ServiceTotalConn, + prometheus.CounterValue, + float64(ipvsSvc.Stats.Connections), + labelValues..., + ) + } + + ch <- prometheus.MustNewConstMetric( + metrics.ControllerIpvsServices, + prometheus.GaugeValue, + float64(len(ipvsSvcs)), + ) +}