diff --git a/dashboard/dashboard.png b/dashboard/dashboard.png
index 3cae5fd7..4c8f1cb5 100644
Binary files a/dashboard/dashboard.png and b/dashboard/dashboard.png differ
diff --git a/dashboard/kube-router.json b/dashboard/kube-router.json
index 5cb4721b..14323760 100644
--- a/dashboard/kube-router.json
+++ b/dashboard/kube-router.json
@@ -1,8 +1,8 @@
{
"__inputs": [
{
- "name": "DS_PROMETHEUS",
- "label": "prometheus",
+ "name": "DS_STAGING",
+ "label": "staging",
"description": "",
"type": "datasource",
"pluginId": "prometheus",
@@ -14,19 +14,25 @@
"type": "grafana",
"id": "grafana",
"name": "Grafana",
- "version": "4.6.2"
+ "version": "5.4.0"
},
{
"type": "panel",
"id": "graph",
"name": "Graph",
- "version": ""
+ "version": "5.0.0"
+ },
+ {
+ "type": "panel",
+ "id": "heatmap",
+ "name": "Heatmap",
+ "version": "5.0.0"
},
{
"type": "datasource",
"id": "prometheus",
"name": "Prometheus",
- "version": "1.0.0"
+ "version": "5.0.0"
}
],
"annotations": {
@@ -45,1412 +51,1526 @@
"editable": true,
"gnetId": null,
"graphTooltip": 1,
- "hideControls": false,
"id": null,
+ "iteration": 1544143515552,
"links": [],
- "refresh": "30s",
- "rows": [
+ "panels": [
{
- "collapse": false,
- "height": 139,
- "panels": [
+ "cards": {
+ "cardPadding": null,
+ "cardRound": null
+ },
+ "color": {
+ "cardColor": "#b4ff00",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateOranges",
+ "exponent": 0.5,
+ "mode": "spectrum"
+ },
+ "dataFormat": "tsbuckets",
+ "datasource": "$datasource",
+ "gridPos": {
+ "h": 5,
+ "w": 8,
+ "x": 0,
+ "y": 0
+ },
+ "heatmap": {},
+ "highlightCards": true,
+ "id": 27,
+ "legend": {
+ "show": false
+ },
+ "links": [],
+ "targets": [
{
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "${DS_PROMETHEUS}",
- "description": "",
- "fill": 1,
- "id": 4,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "hideEmpty": true,
- "hideZero": true,
- "max": false,
- "min": false,
- "rightSide": true,
- "show": false,
- "sort": "max",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(kube_router_service_cps{namespace=~\"[[selnamespace]]\",instance=~\"[[selnode]]\"}) by (namespace, service_name, protocol, port)",
- "format": "time_series",
- "instant": false,
- "intervalFactor": 2,
- "legendFormat": "{{namespace}}/{{service_name}} {{port}}/{{protocol}}",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "Connections",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "label": "Per second",
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "${DS_PROMETHEUS}",
- "fill": 1,
- "id": 13,
- "legend": {
- "alignAsTable": true,
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "rightSide": true,
- "show": false,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(kube_router_service_bps_in{instance=~\"[[selnode]]\"}) by (instance)",
- "format": "time_series",
- "instant": false,
- "intervalFactor": 2,
- "legendFormat": "in: {{instance}}",
- "refId": "A"
- },
- {
- "expr": "- sum(kube_router_service_bps_out{instance=~\"[[selnode]]\"}) by (instance)",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "out: {{instance}}",
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "Traffic by node",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": "Rate",
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "expr": "sum by (le) (increase(kube_router_controller_iptables_sync_time_bucket{instance=~\"$selnode\"}[$interval]))",
+ "format": "heatmap",
+ "intervalFactor": 1,
+ "legendFormat": "{{le}}",
+ "refId": "A"
}
],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": false,
- "title": "Dashboard Row",
- "titleSize": "h3"
+ "title": "Time it took for controller to sync iptables",
+ "tooltip": {
+ "show": true,
+ "showHistogram": false
+ },
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "xBucketNumber": null,
+ "xBucketSize": null,
+ "yAxis": {
+ "decimals": null,
+ "format": "s",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true,
+ "splitFactor": null
+ },
+ "yBucketBound": "auto",
+ "yBucketNumber": null,
+ "yBucketSize": null
},
{
- "collapse": false,
- "height": 158,
- "panels": [
+ "cards": {
+ "cardPadding": null,
+ "cardRound": null
+ },
+ "color": {
+ "cardColor": "#b4ff00",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateReds",
+ "exponent": 0.5,
+ "mode": "spectrum"
+ },
+ "dataFormat": "tsbuckets",
+ "datasource": "$datasource",
+ "gridPos": {
+ "h": 5,
+ "w": 8,
+ "x": 8,
+ "y": 0
+ },
+ "heatmap": {},
+ "highlightCards": true,
+ "id": 28,
+ "legend": {
+ "show": false
+ },
+ "links": [],
+ "targets": [
{
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "${DS_PROMETHEUS}",
- "fill": 1,
- "id": 2,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "hideEmpty": true,
- "hideZero": true,
- "max": false,
- "min": false,
- "rightSide": true,
- "show": false,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(kube_router_service_pps_in{namespace=~\"[[selnamespace]]\",instance=~\"[[selnode]]\"}) by (namespace, service_name, protocol, port)",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "in {{namespace}}/{{service_name}} {{port}}/{{protocol}}",
- "refId": "A"
- },
- {
- "expr": "- sum(kube_router_service_pps_out{namespace=~\"[[selnamespace]]\",instance=~\"[[selnode]]\"}) by (namespace, service_name, protocol, port)",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "out {{namespace}}/{{service_name}} {{port}}/{{protocol}}",
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "Packets",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "pps",
- "label": "Per second",
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "${DS_PROMETHEUS}",
- "fill": 1,
- "id": 7,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "hideEmpty": true,
- "hideZero": true,
- "max": false,
- "min": false,
- "rightSide": true,
- "show": false,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(kube_router_service_bps_in{namespace=~\"[[selnamespace]]\",instance=~\"[[selnode]]\"}) by (namespace, service_name, protocol, port)",
- "format": "time_series",
- "instant": false,
- "intervalFactor": 2,
- "legendFormat": "in {{namespace}}/{{service_name}} {{port}}/{{protocol}}",
- "refId": "A"
- },
- {
- "expr": "- sum(kube_router_service_bps_out{namespace=~\"[[selnamespace]]\",instance=~\"[[selnode]]\"}) by (namespace, service_name, protocol, port)",
- "format": "time_series",
- "instant": false,
- "intervalFactor": 2,
- "legendFormat": "out {{namespace}}/{{service_name}} {{port}}/{{protocol}}",
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "Traffic",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": "Rate",
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- },
- {
- "format": "short",
- "label": "",
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "expr": "sum by (le) (increase(kube_router_controller_ipvs_services_sync_time_bucket{instance=~\"$selnode\"}[$interval]))",
+ "format": "heatmap",
+ "intervalFactor": 1,
+ "legendFormat": "{{le}}",
+ "refId": "A"
}
],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": false,
- "title": "Dashboard Row",
- "titleSize": "h6"
+ "title": "Time it took for controller to sync ipvs services",
+ "tooltip": {
+ "show": true,
+ "showHistogram": false
+ },
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "xBucketNumber": null,
+ "xBucketSize": null,
+ "yAxis": {
+ "decimals": null,
+ "format": "s",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true,
+ "splitFactor": null
+ },
+ "yBucketBound": "auto",
+ "yBucketNumber": null,
+ "yBucketSize": null
},
{
- "collapse": false,
- "height": 164,
- "panels": [
+ "cards": {
+ "cardPadding": null,
+ "cardRound": null
+ },
+ "color": {
+ "cardColor": "#b4ff00",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateViridis",
+ "exponent": 0.5,
+ "mode": "spectrum"
+ },
+ "dataFormat": "tsbuckets",
+ "datasource": "$datasource",
+ "gridPos": {
+ "h": 5,
+ "w": 8,
+ "x": 16,
+ "y": 0
+ },
+ "heatmap": {},
+ "highlightCards": true,
+ "id": 31,
+ "legend": {
+ "show": false
+ },
+ "links": [],
+ "targets": [
{
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "${DS_PROMETHEUS}",
- "fill": 0,
- "id": 16,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": false,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(kube_router_controller_iptables_sync_time{instance=~\"[[selnode]]\"}) by (instance)",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{instance}}",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "Iptables sync",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "s",
- "label": "Time",
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- }
- ]
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "${DS_PROMETHEUS}",
- "fill": 0,
- "id": 18,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": false,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(kube_router_controller_ipvs_services_sync_time{instance=~\"[[selnode]]\"}) by (instance)",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{instance}}",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "Ipvs service sync",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "s",
- "label": "Time",
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- }
- ]
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "${DS_PROMETHEUS}",
- "fill": 0,
- "id": 21,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": false,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(kube_router_controller_bgp_internal_peers_sync_time{instance=~\"[[selnode]]\"}) by (instance)",
- "format": "time_series",
- "intervalFactor": 5,
- "legendFormat": "{{instance}}",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "BGP peer sync",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "s",
- "label": "Time",
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- }
- ]
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "${DS_PROMETHEUS}",
- "fill": 0,
- "id": 23,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": false,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(kube_router_controller_ipvs_metrics_export_time{instance=~\"[[selnode]]\"}) by (instance)",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{instance}}",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "Metrics export time",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "s",
- "label": "Time",
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- }
- ]
+ "expr": "sum by (le) (increase(kube_router_controller_policy_chains_sync_time_bucket{instance=~\"$selnode\"}[$interval]))",
+ "format": "heatmap",
+ "intervalFactor": 1,
+ "legendFormat": "{{le}}",
+ "refId": "A"
}
],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": false,
- "title": "Dashboard Row",
- "titleSize": "h6"
+ "title": "Time it took for controller to sync policy chains",
+ "tooltip": {
+ "show": true,
+ "showHistogram": false
+ },
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "xBucketNumber": null,
+ "xBucketSize": null,
+ "yAxis": {
+ "decimals": null,
+ "format": "s",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true,
+ "splitFactor": null
+ },
+ "yBucketBound": "auto",
+ "yBucketNumber": null,
+ "yBucketSize": null
},
{
- "collapse": false,
- "height": 247,
- "panels": [
+ "cards": {
+ "cardPadding": null,
+ "cardRound": null
+ },
+ "color": {
+ "cardColor": "#b4ff00",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateBlues",
+ "exponent": 0.5,
+ "mode": "spectrum"
+ },
+ "dataFormat": "tsbuckets",
+ "datasource": "$datasource",
+ "gridPos": {
+ "h": 5,
+ "w": 8,
+ "x": 0,
+ "y": 5
+ },
+ "heatmap": {},
+ "highlightCards": true,
+ "id": 30,
+ "legend": {
+ "show": false
+ },
+ "links": [],
+ "targets": [
{
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "${DS_PROMETHEUS}",
- "fill": 0,
- "id": 15,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": false,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(kube_router_controller_ipvs_services{instance=~\"[[selnode]]\"}) by (instance)",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{instance}}",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "Ipvs services",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "none",
- "label": "Per node",
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "${DS_PROMETHEUS}",
- "fill": 0,
- "id": 20,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": false,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "\nsort_desc(sum(kube_router_controller_bgp_peers{instance=~\"[[selnode]]\"}) by (instance))",
- "format": "time_series",
- "instant": false,
- "intervalFactor": 5,
- "legendFormat": "{{instance}}",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "BGP peers 5m avg",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "label": "Peers",
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- }
- ]
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "${DS_PROMETHEUS}",
- "fill": 0,
- "id": 22,
- "legend": {
- "alignAsTable": true,
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "rightSide": true,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(kube_router_controller_bgp_advertisements_received{instance=~\"[[selnode]]\"}[1m])) by (instance)",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{instance}}",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "BGP announcements received 5m avg",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "label": "Rate",
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- }
- ]
+ "expr": "sum by (le) (increase(kube_router_controller_ipvs_metrics_export_time_bucket{instance=~\"$selnode\"}[$interval]))",
+ "format": "heatmap",
+ "intervalFactor": 1,
+ "legendFormat": "{{le}}",
+ "refId": "A"
}
],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": false,
- "title": "Dashboard Row",
- "titleSize": "h6"
+ "title": "Time it took to export metrics",
+ "tooltip": {
+ "show": true,
+ "showHistogram": false
+ },
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "xBucketNumber": null,
+ "xBucketSize": null,
+ "yAxis": {
+ "decimals": null,
+ "format": "s",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true,
+ "splitFactor": null
+ },
+ "yBucketBound": "auto",
+ "yBucketNumber": null,
+ "yBucketSize": null
},
{
- "collapse": false,
- "height": 177,
- "panels": [
+ "cards": {
+ "cardPadding": null,
+ "cardRound": null
+ },
+ "color": {
+ "cardColor": "#b4ff00",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateGreens",
+ "exponent": 0.5,
+ "mode": "spectrum"
+ },
+ "dataFormat": "tsbuckets",
+ "datasource": "$datasource",
+ "gridPos": {
+ "h": 5,
+ "w": 8,
+ "x": 8,
+ "y": 5
+ },
+ "heatmap": {},
+ "highlightCards": true,
+ "id": 29,
+ "legend": {
+ "show": false
+ },
+ "links": [],
+ "targets": [
{
- "aliasColors": {},
- "bars": true,
- "dashLength": 10,
- "dashes": false,
- "datasource": "${DS_PROMETHEUS}",
- "fill": 1,
- "id": 6,
- "legend": {
- "alignAsTable": true,
- "avg": false,
- "current": true,
- "hideEmpty": true,
- "hideZero": true,
- "max": false,
- "min": false,
- "rightSide": true,
- "show": false,
- "sort": null,
- "sortDesc": null,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sort_desc(sum(kube_router_service_bytes_in{namespace=~\"[[selnamespace]]\",instance=~\"[[selnode]]\"} > 0) by (namespace, service_name, protocol, port))",
- "format": "time_series",
- "instant": true,
- "intervalFactor": 2,
- "legendFormat": "{{namespace}}
{{service_name}}
{{port}}/{{protocol}}",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "Bytes In",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "series",
- "name": null,
- "show": true,
- "values": [
- "total"
- ]
- },
- "yaxes": [
- {
- "decimals": null,
- "format": "decbytes",
- "label": "Total",
- "logBase": 10,
- "max": null,
- "min": null,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": {},
- "bars": true,
- "dashLength": 10,
- "dashes": false,
- "datasource": "${DS_PROMETHEUS}",
- "fill": 1,
- "id": 9,
- "legend": {
- "alignAsTable": true,
- "avg": false,
- "current": true,
- "hideEmpty": false,
- "hideZero": false,
- "max": false,
- "min": false,
- "rightSide": true,
- "show": false,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": false,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sort_desc(sum(kube_router_service_bytes_out{namespace=~\"[[selnamespace]]\",instance=~\"[[selnode]]\"} > 0) by (namespace, service_name, protocol, port))",
- "format": "time_series",
- "instant": true,
- "intervalFactor": 2,
- "legendFormat": "{{namespace}}
{{service_name}}
{{port}}/{{protocol}}",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "Bytes Out",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "series",
- "name": null,
- "show": true,
- "values": [
- "current"
- ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": "Total",
- "logBase": 10,
- "max": null,
- "min": null,
- "show": true
- },
- {
- "format": "bytes",
- "label": null,
- "logBase": 2,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "expr": "sum by (le) (increase(kube_router_controller_bgp_internal_peers_sync_time_bucket{instance=~\"$selnode\"}[$interval]))",
+ "format": "heatmap",
+ "intervalFactor": 1,
+ "legendFormat": "{{le}}",
+ "refId": "A"
}
],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": false,
- "title": "Dashboard Row",
- "titleSize": "h6"
+ "title": "Time it took to sync internal bgp peers",
+ "tooltip": {
+ "show": true,
+ "showHistogram": false
+ },
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "xBucketNumber": null,
+ "xBucketSize": null,
+ "yAxis": {
+ "decimals": null,
+ "format": "s",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true,
+ "splitFactor": null
+ },
+ "yBucketBound": "auto",
+ "yBucketNumber": null,
+ "yBucketSize": null
},
{
- "collapse": false,
- "height": 154,
- "panels": [
+ "cards": {
+ "cardPadding": null,
+ "cardRound": null
+ },
+ "color": {
+ "cardColor": "#b4ff00",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolatePurples",
+ "exponent": 0.5,
+ "mode": "spectrum"
+ },
+ "dataFormat": "tsbuckets",
+ "datasource": "$datasource",
+ "gridPos": {
+ "h": 5,
+ "w": 8,
+ "x": 16,
+ "y": 5
+ },
+ "heatmap": {},
+ "highlightCards": true,
+ "id": 32,
+ "legend": {
+ "show": false
+ },
+ "links": [],
+ "targets": [
{
- "aliasColors": {},
- "bars": true,
- "dashLength": 10,
- "dashes": false,
- "datasource": "${DS_PROMETHEUS}",
- "fill": 1,
- "id": 10,
- "legend": {
- "alignAsTable": true,
- "avg": false,
- "current": true,
- "hideEmpty": true,
- "hideZero": true,
- "max": false,
- "min": false,
- "rightSide": true,
- "show": false,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": false,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sort_desc(sum(kube_router_service_packets_in{namespace=~\"[[selnamespace]]\",instance=~\"[[selnode]]\"} > 0) by (namespace, service_name, protocol, port))",
- "format": "time_series",
- "instant": true,
- "intervalFactor": 2,
- "legendFormat": "{{namespace}}
{{service_name}}
{{port}}/{{protocol}}",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "Packets In",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "series",
- "name": null,
- "show": true,
- "values": [
- "total"
- ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": "Total",
- "logBase": 10,
- "max": null,
- "min": null,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": {},
- "bars": true,
- "dashLength": 10,
- "dashes": false,
- "datasource": "${DS_PROMETHEUS}",
- "fill": 1,
- "id": 11,
- "legend": {
- "alignAsTable": true,
- "avg": false,
- "current": true,
- "hideEmpty": true,
- "hideZero": true,
- "max": false,
- "min": false,
- "rightSide": true,
- "show": false,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": false,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sort_desc(sum(kube_router_service_packets_out{namespace=~\"[[selnamespace]]\",instance=~\"[[selnode]]\"} > 0) by (namespace, service_name, protocol, port))",
- "format": "time_series",
- "instant": true,
- "intervalFactor": 2,
- "legendFormat": "{{namespace}}
{{service_name}}
{{port}}/{{protocol}}",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "Packets Out",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "series",
- "name": null,
- "show": true,
- "values": [
- "total"
- ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": "Total",
- "logBase": 10,
- "max": null,
- "min": null,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "expr": "sum by (le) (increase(kube_router_controller_routes_sync_time_bucket{instance=~\"$selnode\"}[$interval]))",
+ "format": "heatmap",
+ "intervalFactor": 1,
+ "legendFormat": "{{le}}",
+ "refId": "A"
}
],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": false,
- "title": "Dashboard Row",
- "titleSize": "h6"
+ "title": "Time it took to sync internal bgp peers",
+ "tooltip": {
+ "show": true,
+ "showHistogram": false
+ },
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "xBucketNumber": null,
+ "xBucketSize": null,
+ "yAxis": {
+ "decimals": null,
+ "format": "s",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true,
+ "splitFactor": null
+ },
+ "yBucketBound": "auto",
+ "yBucketNumber": null,
+ "yBucketSize": null
},
{
- "collapse": false,
- "height": 250,
- "panels": [
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fill": 1,
+ "gridPos": {
+ "h": 4,
+ "w": 12,
+ "x": 0,
+ "y": 10
+ },
+ "id": 2,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "hideEmpty": true,
+ "hideZero": true,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": false,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
{
- "aliasColors": {},
- "bars": true,
- "dashLength": 10,
- "dashes": false,
- "datasource": "${DS_PROMETHEUS}",
- "description": "Total connections made to the service",
- "fill": 1,
- "id": 12,
- "legend": {
- "alignAsTable": true,
- "avg": false,
- "current": true,
- "hideEmpty": true,
- "hideZero": true,
- "max": false,
- "min": false,
- "rightSide": true,
- "show": false,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": false,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "span": 12,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sort_desc(sum(kube_router_service_total_connections{namespace=~\"[[selnamespace]]\",instance=~\"[[selnode]]\"} > 0) by (namespace, service_name, protocol, port))",
- "format": "time_series",
- "instant": true,
- "intervalFactor": 3,
- "legendFormat": "{{namespace}}
{{service_name}}
{{port}}/{{protocol}}",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "Connections",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "series",
- "name": null,
- "show": true,
- "values": [
- "total"
- ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": "Total",
- "logBase": 10,
- "max": null,
- "min": null,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "expr": "sum(kube_router_service_pps_in{svc_namespace=~\"[[selnamespace]]\",instance=~\"[[selnode]]\"}) by (svc_namespace, service_name, protocol, port)",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "in {{svc_namespace}}/{{service_name}} {{port}}/{{protocol}}",
+ "refId": "A"
+ },
+ {
+ "expr": "- sum(kube_router_service_pps_out{svc_namespace=~\"[[selnamespace]]\",instance=~\"[[selnode]]\"}) by (svc_namespace, service_name, protocol, port)",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "out {{svc_namespace}}/{{service_name}} {{port}}/{{protocol}}",
+ "refId": "B"
}
],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": false,
- "title": "Dashboard Row",
- "titleSize": "h6"
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Packets",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "pps",
+ "label": "Per second",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fill": 1,
+ "gridPos": {
+ "h": 4,
+ "w": 12,
+ "x": 12,
+ "y": 10
+ },
+ "id": 13,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(kube_router_service_bps_in{instance=~\"[[selnode]]\"}) by (instance)",
+ "format": "time_series",
+ "instant": false,
+ "intervalFactor": 2,
+ "legendFormat": "in: {{instance}}",
+ "refId": "A"
+ },
+ {
+ "expr": "- sum(kube_router_service_bps_out{instance=~\"[[selnode]]\"}) by (instance)",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "out: {{instance}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Traffic by node",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "Bps",
+ "label": "Rate",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fill": 0,
+ "gridPos": {
+ "h": 5,
+ "w": 8,
+ "x": 0,
+ "y": 14
+ },
+ "id": 15,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(kube_router_controller_ipvs_services{instance=~\"[[selnode]]\"}) by (instance)",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Ipvs services",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "none",
+ "label": "Per node",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fill": 0,
+ "gridPos": {
+ "h": 5,
+ "w": 8,
+ "x": 8,
+ "y": 14
+ },
+ "id": 20,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "\nsort_desc(sum(kube_router_controller_bgp_peers{instance=~\"[[selnode]]\"}) by (instance))",
+ "format": "time_series",
+ "instant": false,
+ "intervalFactor": 1,
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "BGP peers 5m avg",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": "Peers",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fill": 0,
+ "gridPos": {
+ "h": 5,
+ "w": 8,
+ "x": 16,
+ "y": 14
+ },
+ "id": 22,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(increase(kube_router_controller_bgp_advertisements_received{instance=~\"[[selnode]]\"}[$interval])) by (instance)",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{instance}} received",
+ "refId": "A"
+ },
+ {
+ "expr": "-sum(increase(kube_router_controller_bgp_advertisements_sent{instance=~\"[[selnode]]\"}[$interval])) by (instance)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{instance}} sent",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "BGP announcements",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fill": 1,
+ "gridPos": {
+ "h": 5,
+ "w": 12,
+ "x": 0,
+ "y": 19
+ },
+ "id": 6,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": true,
+ "hideEmpty": true,
+ "hideZero": true,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": false,
+ "sort": null,
+ "sortDesc": null,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(increase(kube_router_service_bytes_in{svc_namespace=~\"[[selnamespace]]\",instance=~\"[[selnode]]\"}[$interval])) by (svc_namespace, service_name, protocol, port)",
+ "format": "time_series",
+ "instant": false,
+ "intervalFactor": 1,
+ "legendFormat": "{{svc_namespace}}/{{service_name}} {{port}} {{protocol}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Bytes In",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "decimals": null,
+ "format": "decbytes",
+ "label": "Total",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fill": 1,
+ "gridPos": {
+ "h": 4,
+ "w": 12,
+ "x": 12,
+ "y": 19
+ },
+ "id": 7,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "hideEmpty": true,
+ "hideZero": true,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": false,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(kube_router_service_bps_in{svc_namespace=~\"[[selnamespace]]\",instance=~\"[[selnode]]\"}) by (svc_namespace, service_name, protocol, port)",
+ "format": "time_series",
+ "instant": false,
+ "intervalFactor": 1,
+ "legendFormat": "in {{svc_namespace}}/{{service_name}} {{port}}/{{protocol}}",
+ "refId": "A"
+ },
+ {
+ "expr": "- sum(kube_router_service_bps_out{svc_namespace=~\"[[selnamespace]]\",instance=~\"[[selnode]]\"}) by (svc_namespace, service_name, protocol, port)",
+ "format": "time_series",
+ "instant": false,
+ "intervalFactor": 1,
+ "legendFormat": "out {{svc_namespace}}/{{service_name}} {{port}}/{{protocol}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Traffic",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "Bps",
+ "label": "Rate",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fill": 1,
+ "gridPos": {
+ "h": 5,
+ "w": 12,
+ "x": 12,
+ "y": 23
+ },
+ "id": 9,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": true,
+ "hideEmpty": true,
+ "hideZero": true,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": false,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(increase(kube_router_service_bytes_out{svc_namespace=~\"[[selnamespace]]\",instance=~\"[[selnode]]\"}[$interval])) by (svc_namespace, service_name, protocol, port)",
+ "format": "time_series",
+ "instant": false,
+ "intervalFactor": 1,
+ "legendFormat": "{{svc_namespace}}/{{service_name}} {{port}} {{protocol}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Bytes Out",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "bytes",
+ "label": "Total",
+ "logBase": 10,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "bytes",
+ "label": null,
+ "logBase": 2,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fill": 1,
+ "gridPos": {
+ "h": 4,
+ "w": 12,
+ "x": 0,
+ "y": 24
+ },
+ "id": 10,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": true,
+ "hideEmpty": true,
+ "hideZero": true,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": false,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(increase(kube_router_service_packets_in{svc_namespace=~\"[[selnamespace]]\",instance=~\"[[selnode]]\"}[$interval])) by (svc_namespace, service_name, protocol, port)",
+ "format": "time_series",
+ "instant": false,
+ "intervalFactor": 1,
+ "legendFormat": "{{svc_namespace}}/{{service_name}} {{port}} {{protocol}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Packets In",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "pps",
+ "label": "Total",
+ "logBase": 10,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "description": "Total connections made to the service",
+ "fill": 1,
+ "gridPos": {
+ "h": 4,
+ "w": 12,
+ "x": 0,
+ "y": 28
+ },
+ "id": 12,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": false,
+ "hideEmpty": true,
+ "hideZero": true,
+ "max": false,
+ "min": false,
+ "rightSide": false,
+ "show": false,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum by (svc_namespace, service_name, port, protocol) (increase(kube_router_service_total_connections{svc_namespace=~\"$selnamespace\",instance=~\"$selnode\"}[$interval]))",
+ "format": "time_series",
+ "instant": false,
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{svc_namespace}}/{{service_name}} {{port}} {{protocol}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Connections",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": "Total",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fill": 1,
+ "gridPos": {
+ "h": 4,
+ "w": 12,
+ "x": 12,
+ "y": 28
+ },
+ "id": 11,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": true,
+ "hideEmpty": true,
+ "hideZero": true,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": false,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(increase(kube_router_service_packets_out{svc_namespace=~\"[[selnamespace]]\",instance=~\"[[selnode]]\"}[$interval])) by (svc_namespace, service_name, protocol, port)",
+ "format": "time_series",
+ "instant": false,
+ "intervalFactor": 1,
+ "legendFormat": "{{svc_namespace}}/{{service_name}} {{port}} {{protocol}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Packets Out",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": "Total",
+ "logBase": 10,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "description": "",
+ "fill": 1,
+ "gridPos": {
+ "h": 6,
+ "w": 24,
+ "x": 0,
+ "y": 32
+ },
+ "id": 4,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "hideEmpty": true,
+ "hideZero": true,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": false,
+ "sort": "max",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(kube_router_service_cps{svc_namespace=~\"[[selnamespace]]\",instance=~\"[[selnode]]\"}) by (svc_namespace, service_name, protocol, port)",
+ "format": "time_series",
+ "instant": false,
+ "intervalFactor": 1,
+ "legendFormat": "{{svc_namespace}}/{{service_name}} {{port}}/{{protocol}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Connections per second",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
}
],
- "schemaVersion": 14,
+ "refresh": "30s",
+ "schemaVersion": 16,
"style": "dark",
"tags": [
+ "Kubernetes",
"kube-router",
- "network"
+ "networking"
],
"templating": {
"list": [
+ {
+ "current": {
+ "text": "staging",
+ "value": "staging"
+ },
+ "hide": 0,
+ "label": "Datasource",
+ "name": "datasource",
+ "options": [],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "type": "datasource"
+ },
{
"allValue": ".*",
"current": {},
- "datasource": "${DS_PROMETHEUS}",
+ "datasource": "${DS_STAGING}",
+ "definition": "label_values(kube_router_service_total_connections, svc_namespace)",
"hide": 0,
"includeAll": true,
"label": "Namespace",
"multi": true,
"name": "selnamespace",
"options": [],
- "query": "label_values(kube_router_service_total_connections, namespace)",
+ "query": "label_values(kube_router_service_total_connections, svc_namespace)",
"refresh": 1,
"regex": "",
+ "skipUrlSync": false,
"sort": 1,
"tagValuesQuery": "",
"tags": [],
@@ -1461,7 +1581,8 @@
{
"allValue": ".*",
"current": {},
- "datasource": "${DS_PROMETHEUS}",
+ "datasource": "${DS_STAGING}",
+ "definition": "",
"hide": 0,
"includeAll": true,
"label": "Instance",
@@ -1471,17 +1592,86 @@
"query": "label_values(kube_router_service_total_connections, instance)",
"refresh": 1,
"regex": "",
+ "skipUrlSync": false,
"sort": 1,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
+ },
+ {
+ "auto": false,
+ "auto_count": 30,
+ "auto_min": "10s",
+ "current": {
+ "text": "1m",
+ "value": "1m"
+ },
+ "hide": 0,
+ "label": "Interval",
+ "name": "interval",
+ "options": [
+ {
+ "selected": true,
+ "text": "1m",
+ "value": "1m"
+ },
+ {
+ "selected": false,
+ "text": "10m",
+ "value": "10m"
+ },
+ {
+ "selected": false,
+ "text": "30m",
+ "value": "30m"
+ },
+ {
+ "selected": false,
+ "text": "1h",
+ "value": "1h"
+ },
+ {
+ "selected": false,
+ "text": "6h",
+ "value": "6h"
+ },
+ {
+ "selected": false,
+ "text": "12h",
+ "value": "12h"
+ },
+ {
+ "selected": false,
+ "text": "1d",
+ "value": "1d"
+ },
+ {
+ "selected": false,
+ "text": "7d",
+ "value": "7d"
+ },
+ {
+ "selected": false,
+ "text": "14d",
+ "value": "14d"
+ },
+ {
+ "selected": false,
+ "text": "30d",
+ "value": "30d"
+ }
+ ],
+ "query": "1m,10m,30m,1h,6h,12h,1d,7d,14d,30d",
+ "refresh": 2,
+ "skipUrlSync": false,
+ "type": "interval"
}
]
},
"time": {
- "from": "now-5m",
+ "from": "now-1h",
"to": "now"
},
"timepicker": {
@@ -1511,5 +1701,6 @@
},
"timezone": "browser",
"title": "kube-router",
- "version": 48
-}
+ "uid": "Xw9Eyxciz",
+ "version": 16
+}
\ No newline at end of file
diff --git a/docs/metrics.md b/docs/metrics.md
index d9a97782..39a85acd 100644
--- a/docs/metrics.md
+++ b/docs/metrics.md
@@ -22,6 +22,8 @@ The default values unless other specified is
By enabling [Kubernetes SD](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#) in Prometheus configuration & adding required annotations Prometheus can automaticly discover & scrape kube-router metrics
## Version notes
+kube-router v0.2.4 received a metrics overhaul where some metrics were changed into histograms, additional metrics was also added. Please make sure you are using the latest dashboard version with versions => v0.2.4
+
kube-router 0.1.0-rc2 and upwards supports the runtime configuration for controlling where to expose the metrics. If you are using a older version, metrics path & port is locked to `/metrics` & `8080`
## Supported annotations
@@ -56,14 +58,20 @@ The following metrics is exposed by kube-router prefixed by `kube_router_`
* controller_bgp_peers
Number of BGP peers of the instance
* controller_bgp_advertisements_received
- Number of total BGP advertisements received since kube-router start
+ Total number of BGP advertisements received since kube-router started
+* controller_bgp_advertisements_sent
+ Total number of BGP advertisements sent since kube-router started
* controller_bgp_internal_peers_sync_time
Time it took for the BGP internal peer sync loop to complete
+* controller_routes_sync_time
+ Time it took for controller to sync routes
### run-firewall=true
* controller_iptables_sync_time
Time it took for the iptables sync loop to complete
+* controller_policy_chains_sync_time
+ Time it took for controller to sync policy chains
### run-service-proxy = true
@@ -95,7 +103,7 @@ The following metrics is exposed by kube-router prefixed by `kube_router_`
Outgoing bytes per second
To get a grouped list of CPS for each service a Prometheus query could look like this e.g:
-`sum(kube_router_service_cps) by (namespace, service_name)`
+`sum(kube_router_service_cps) by (svc_namespace, service_name)`
## Grafana Dashboard
diff --git a/pkg/controllers/netpol/network_policy_controller.go b/pkg/controllers/netpol/network_policy_controller.go
index 7cc67f9a..56bf0e7d 100644
--- a/pkg/controllers/netpol/network_policy_controller.go
+++ b/pkg/controllers/netpol/network_policy_controller.go
@@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"net"
+ "regexp"
"strconv"
"strings"
"sync"
@@ -26,7 +27,6 @@ import (
"k8s.io/client-go/kubernetes"
listers "k8s.io/client-go/listers/core/v1"
"k8s.io/client-go/tools/cache"
- "regexp"
)
const (
@@ -215,7 +215,7 @@ func (npc *NetworkPolicyController) Sync() error {
defer func() {
endTime := time.Since(start)
if npc.MetricsEnabled {
- metrics.ControllerIptablesSyncTime.WithLabelValues().Set(float64(endTime.Seconds()))
+ metrics.ControllerIptablesSyncTime.Observe(endTime.Seconds())
}
glog.V(1).Infof("sync iptables took %v", endTime)
}()
@@ -258,7 +258,12 @@ func (npc *NetworkPolicyController) Sync() error {
// policyspec is evaluated to set of matching pods, which are grouped in to a
// ipset used for source ip addr matching.
func (npc *NetworkPolicyController) syncNetworkPolicyChains(version string) (map[string]bool, map[string]bool, error) {
-
+ start := time.Now()
+ defer func() {
+ endTime := time.Since(start)
+ metrics.ControllerPolicyChainsSyncTime.Observe(endTime.Seconds())
+ glog.V(2).Infof("Syncing network policy chains took %v", endTime)
+ }()
activePolicyChains := make(map[string]bool)
activePolicyIpSets := make(map[string]bool)
@@ -1536,6 +1541,7 @@ func NewNetworkPolicyController(clientset kubernetes.Interface,
if config.MetricsEnabled {
//Register the metrics for this controller
prometheus.MustRegister(metrics.ControllerIptablesSyncTime)
+ prometheus.MustRegister(metrics.ControllerPolicyChainsSyncTime)
npc.MetricsEnabled = true
}
diff --git a/pkg/controllers/proxy/network_services_controller.go b/pkg/controllers/proxy/network_services_controller.go
index 2b08237a..d7dda2cb 100644
--- a/pkg/controllers/proxy/network_services_controller.go
+++ b/pkg/controllers/proxy/network_services_controller.go
@@ -373,7 +373,9 @@ func (nsc *NetworkServicesController) publishMetrics(serviceInfoMap serviceInfoM
defer func() {
endTime := time.Since(start)
glog.V(2).Infof("Publishing IPVS metrics took %v", endTime)
- metrics.ControllerIpvsMetricsExportTime.WithLabelValues().Set(float64(endTime.Seconds()))
+ if nsc.MetricsEnabled {
+ metrics.ControllerIpvsMetricsExportTime.Observe(float64(endTime.Seconds()))
+ }
}()
ipvsSvcs, err := nsc.ln.ipvsGetServices()
@@ -429,7 +431,7 @@ func (nsc *NetworkServicesController) publishMetrics(serviceInfoMap serviceInfoM
metrics.ServicePpsIn.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.PPSIn))
metrics.ServicePpsOut.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.PPSOut))
metrics.ServiceTotalConn.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.Connections))
- metrics.ControllerIpvsServices.WithLabelValues().Set(float64(len(ipvsSvcs)))
+ metrics.ControllerIpvsServices.Set(float64(len(ipvsSvcs)))
}
}
}
@@ -528,7 +530,7 @@ func (nsc *NetworkServicesController) syncIpvsServices(serviceInfoMap serviceInf
defer func() {
endTime := time.Since(start)
if nsc.MetricsEnabled {
- metrics.ControllerIpvsServicesSyncTime.WithLabelValues().Set(float64(endTime.Seconds()))
+ metrics.ControllerIpvsServicesSyncTime.Observe(endTime.Seconds())
}
glog.V(1).Infof("sync ipvs services took %v", endTime)
}()
diff --git a/pkg/controllers/routing/bgp_peers.go b/pkg/controllers/routing/bgp_peers.go
index a120011e..abeda348 100644
--- a/pkg/controllers/routing/bgp_peers.go
+++ b/pkg/controllers/routing/bgp_peers.go
@@ -30,7 +30,9 @@ func (nrc *NetworkRoutingController) syncInternalPeers() {
start := time.Now()
defer func() {
endTime := time.Since(start)
- metrics.ControllerBGPInternalPeersSyncTime.WithLabelValues().Set(float64(endTime.Seconds()))
+ if nrc.MetricsEnabled {
+ metrics.ControllerBGPInternalPeersSyncTime.Observe(endTime.Seconds())
+ }
glog.V(2).Infof("Syncing BGP peers for the node took %v", endTime)
}()
@@ -40,8 +42,9 @@ func (nrc *NetworkRoutingController) syncInternalPeers() {
glog.Errorf("Failed to list nodes from API server due to: %s. Can not perform BGP peer sync", err.Error())
return
}
-
- metrics.ControllerBPGpeers.WithLabelValues().Set(float64(len(nodes.Items)))
+ if nrc.MetricsEnabled {
+ metrics.ControllerBPGpeers.Set(float64(len(nodes.Items)))
+ }
// establish peer and add Pod CIDRs with current set of nodes
currentNodes := make([]string, 0)
for _, node := range nodes.Items {
diff --git a/pkg/controllers/routing/network_routes_controller.go b/pkg/controllers/routing/network_routes_controller.go
index b4b02781..137aad47 100644
--- a/pkg/controllers/routing/network_routes_controller.go
+++ b/pkg/controllers/routing/network_routes_controller.go
@@ -325,7 +325,7 @@ func (nrc *NetworkRoutingController) watchBgpUpdates() {
case *gobgp.WatchEventBestPath:
glog.V(3).Info("Processing bgp route advertisement from peer")
if nrc.MetricsEnabled {
- metrics.ControllerBGPadvertisementsReceived.WithLabelValues().Add(float64(1))
+ metrics.ControllerBGPadvertisementsReceived.Inc()
}
for _, path := range msg.PathList {
if path.IsLocal() {
@@ -342,6 +342,9 @@ func (nrc *NetworkRoutingController) watchBgpUpdates() {
}
func (nrc *NetworkRoutingController) advertisePodRoute() error {
+ if nrc.MetricsEnabled {
+ metrics.ControllerBGPadvertisementsSent.Inc()
+ }
cidr, err := utils.GetPodCidrFromNodeSpec(nrc.clientset, nrc.hostnameOverride)
if err != nil {
return err
@@ -486,6 +489,12 @@ func (nrc *NetworkRoutingController) Cleanup() {
}
func (nrc *NetworkRoutingController) syncNodeIPSets() error {
+ start := time.Now()
+ defer func() {
+ if nrc.MetricsEnabled {
+ metrics.ControllerRoutesSyncTime.Observe(time.Since(start).Seconds())
+ }
+ }()
// Get the current list of the nodes from API server
nodes, err := nrc.clientset.CoreV1().Nodes().List(metav1.ListOptions{})
if err != nil {
@@ -786,6 +795,7 @@ func NewNetworkRoutingController(clientset kubernetes.Interface,
prometheus.MustRegister(metrics.ControllerBGPadvertisementsReceived)
prometheus.MustRegister(metrics.ControllerBGPInternalPeersSyncTime)
prometheus.MustRegister(metrics.ControllerBPGpeers)
+ prometheus.MustRegister(metrics.ControllerRoutesSyncTime)
nrc.MetricsEnabled = true
}
diff --git a/pkg/metrics/metrics_controller.go b/pkg/metrics/metrics_controller.go
index 43a74ad8..a3cc0c38 100644
--- a/pkg/metrics/metrics_controller.go
+++ b/pkg/metrics/metrics_controller.go
@@ -21,100 +21,130 @@ const (
)
var (
+ // ServiceTotalConn Total incoming connections made
ServiceTotalConn = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Name: "service_total_connections",
Help: "Total incoming connections made",
- }, []string{"namespace", "service_name", "service_vip", "protocol", "port"})
+ }, []string{"svc_namespace", "service_name", "service_vip", "protocol", "port"})
+ // ServicePacketsIn Total incoming packets
ServicePacketsIn = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Name: "service_packets_in",
Help: "Total incoming packets",
- }, []string{"namespace", "service_name", "service_vip", "protocol", "port"})
+ }, []string{"svc_namespace", "service_name", "service_vip", "protocol", "port"})
+ // ServicePacketsOut Total outgoing packets
ServicePacketsOut = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Name: "service_packets_out",
Help: "Total outgoing packets",
- }, []string{"namespace", "service_name", "service_vip", "protocol", "port"})
+ }, []string{"svc_namespace", "service_name", "service_vip", "protocol", "port"})
+ // ServiceBytesIn Total incoming bytes
ServiceBytesIn = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Name: "service_bytes_in",
Help: "Total incoming bytes",
- }, []string{"namespace", "service_name", "service_vip", "protocol", "port"})
+ }, []string{"svc_namespace", "service_name", "service_vip", "protocol", "port"})
+ // ServiceBytesOut Total outgoing bytes
ServiceBytesOut = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Name: "service_bytes_out",
Help: "Total outgoing bytes",
- }, []string{"namespace", "service_name", "service_vip", "protocol", "port"})
+ }, []string{"svc_namespace", "service_name", "service_vip", "protocol", "port"})
+ // ServicePpsIn Incoming packets per second
ServicePpsIn = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Name: "service_pps_in",
Help: "Incoming packets per second",
- }, []string{"namespace", "service_name", "service_vip", "protocol", "port"})
+ }, []string{"svc_namespace", "service_name", "service_vip", "protocol", "port"})
+ // ServicePpsOut Outgoing packets per second
ServicePpsOut = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Name: "service_pps_out",
Help: "Outgoing packets per second",
- }, []string{"namespace", "service_name", "service_vip", "protocol", "port"})
+ }, []string{"svc_namespace", "service_name", "service_vip", "protocol", "port"})
+ // ServiceCPS Service connections per second
ServiceCPS = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Name: "service_cps",
Help: "Service connections per second",
- }, []string{"namespace", "service_name", "service_vip", "protocol", "port"})
+ }, []string{"svc_namespace", "service_name", "service_vip", "protocol", "port"})
+ // ServiceBpsIn Incoming bytes per second
ServiceBpsIn = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Name: "service_bps_in",
Help: "Incoming bytes per second",
- }, []string{"namespace", "service_name", "service_vip", "protocol", "port"})
+ }, []string{"svc_namespace", "service_name", "service_vip", "protocol", "port"})
+ // ServiceBpsOut Outgoing bytes per second
ServiceBpsOut = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Name: "service_bps_out",
Help: "Outgoing bytes per second",
- }, []string{"namespace", "service_name", "service_vip", "protocol", "port"})
- ControllerIpvsServices = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+ }, []string{"svc_namespace", "service_name", "service_vip", "protocol", "port"})
+ // ControllerIpvsServices Number of ipvs services in the instance
+ ControllerIpvsServices = prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: namespace,
Name: "controller_ipvs_services",
Help: "Number of ipvs services in the instance",
- }, []string{})
- ControllerIptablesSyncTime = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+ })
+ // ControllerIptablesSyncTime Time it took for controller to sync iptables
+ ControllerIptablesSyncTime = prometheus.NewHistogram(prometheus.HistogramOpts{
Namespace: namespace,
Name: "controller_iptables_sync_time",
Help: "Time it took for controller to sync iptables",
- }, []string{})
- ControllerPublishMetricsTime = prometheus.NewGaugeVec(prometheus.GaugeOpts{
- Namespace: namespace,
- Name: "controller_publish_metrics_time",
- Help: "Time it took to publish metrics",
- }, []string{})
- ControllerIpvsServicesSyncTime = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+ })
+ // ControllerIpvsServicesSyncTime Time it took for controller to sync ipvs services
+ ControllerIpvsServicesSyncTime = prometheus.NewHistogram(prometheus.HistogramOpts{
Namespace: namespace,
Name: "controller_ipvs_services_sync_time",
Help: "Time it took for controller to sync ipvs services",
- }, []string{})
- ControllerBPGpeers = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+ })
+ // ControllerRoutesSyncTime Time it took for controller to sync ipvs services
+ ControllerRoutesSyncTime = prometheus.NewHistogram(prometheus.HistogramOpts{
+ Namespace: namespace,
+ Name: "controller_routes_sync_time",
+ Help: "Time it took for controller to sync routes",
+ })
+ // ControllerBPGpeers BGP peers in the runtime configuration
+ ControllerBPGpeers = prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: namespace,
Name: "controller_bgp_peers",
Help: "BGP peers in the runtime configuration",
- }, []string{})
- ControllerBGPInternalPeersSyncTime = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+ })
+ // ControllerBGPInternalPeersSyncTime Time it took to sync internal bgp peers
+ ControllerBGPInternalPeersSyncTime = prometheus.NewHistogram(prometheus.HistogramOpts{
Namespace: namespace,
Name: "controller_bgp_internal_peers_sync_time",
Help: "Time it took to sync internal bgp peers",
- }, []string{})
- ControllerBGPadvertisementsReceived = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+ })
+ // ControllerBGPadvertisementsReceived Time it took to sync internal bgp peers
+ ControllerBGPadvertisementsReceived = prometheus.NewCounter(prometheus.CounterOpts{
Namespace: namespace,
Name: "controller_bgp_advertisements_received",
- Help: "Time it took to sync internal bgp peers",
- }, []string{})
- ControllerIpvsMetricsExportTime = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+ Help: "BGP advertisements received",
+ })
+ // ControllerBGPadvertisementsSent Time it took to sync internal bgp peers
+ ControllerBGPadvertisementsSent = prometheus.NewCounter(prometheus.CounterOpts{
+ Namespace: namespace,
+ Name: "controller_bgp_advertisements_sent",
+ Help: "BGP advertisements sent",
+ })
+ // ControllerIpvsMetricsExportTime Time it took to export metrics
+ ControllerIpvsMetricsExportTime = prometheus.NewHistogram(prometheus.HistogramOpts{
Namespace: namespace,
Name: "controller_ipvs_metrics_export_time",
Help: "Time it took to export metrics",
- }, []string{})
+ })
+ // ControllerPolicyChainsSyncTime Time it took for controller to sync policys
+ ControllerPolicyChainsSyncTime = prometheus.NewHistogram(prometheus.HistogramOpts{
+ Namespace: namespace,
+ Name: "controller_policy_chains_sync_time",
+ Help: "Time it took for controller to sync policy chains",
+ })
)
-// MetricsController Holds settings for the metrics controller
-type MetricsController struct {
+// Controller Holds settings for the metrics controller
+type Controller struct {
MetricsPath string
MetricsPort uint16
mu sync.Mutex
@@ -122,7 +152,7 @@ type MetricsController struct {
}
// Run prometheus metrics controller
-func (mc *MetricsController) Run(healthChan chan<- *healthcheck.ControllerHeartbeat, stopCh <-chan struct{}, wg *sync.WaitGroup) error {
+func (mc *Controller) Run(healthChan chan<- *healthcheck.ControllerHeartbeat, stopCh <-chan struct{}, wg *sync.WaitGroup) error {
t := time.NewTicker(3 * time.Second)
defer wg.Done()
glog.Info("Starting metrics controller")
@@ -157,8 +187,8 @@ func (mc *MetricsController) Run(healthChan chan<- *healthcheck.ControllerHeartb
}
// NewMetricsController returns new MetricController object
-func NewMetricsController(clientset kubernetes.Interface, config *options.KubeRouterConfig) (*MetricsController, error) {
- mc := MetricsController{}
+func NewMetricsController(clientset kubernetes.Interface, config *options.KubeRouterConfig) (*Controller, error) {
+ mc := Controller{}
mc.MetricsPath = config.MetricsPath
mc.MetricsPort = config.MetricsPort
return &mc, nil