From 37feb034cd294affc8c0b2eb6e7b4d29e57c9508 Mon Sep 17 00:00:00 2001 From: Frederic Branczyk Date: Tue, 21 Feb 2017 14:06:53 +0100 Subject: [PATCH 1/3] make node gauges usage based --- assets/grafana/node-dashboard.json | 37 +++++++++++++++++++----------- manifests/grafana/grafana-cm.yaml | 37 +++++++++++++++++++----------- 2 files changed, 46 insertions(+), 28 deletions(-) diff --git a/assets/grafana/node-dashboard.json b/assets/grafana/node-dashboard.json index bac1eb0d..19b5ba2d 100644 --- a/assets/grafana/node-dashboard.json +++ b/assets/grafana/node-dashboard.json @@ -307,17 +307,26 @@ { "expr": "node_memory_MemTotal{instance=~\"$server\"} - node_memory_MemFree{instance=~\"$server\"}", "intervalFactor": 2, - "legendFormat": "free memory", + "legendFormat": "memory usage", "metric": "memo", "refId": "A", "step": 4, "target": "" + }, + { + "expr": "node_memory_MemTotal{instance=~\"$server\"}", + "intervalFactor": 2, + "legendFormat": "memory total", + "metric": "memo", + "refId": "B", + "step": 4, + "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Free memory", + "title": "Memory usage", "tooltip": { "msResolution": false, "shared": true, @@ -352,9 +361,9 @@ "colorBackground": false, "colorValue": false, "colors": [ - "rgba(245, 54, 54, 0.9)", + "rgba(50, 172, 45, 0.97)", "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" + "rgba(245, 54, 54, 0.9)" ], "datasource": "${DS_PROMETHEUS}", "editable": true, @@ -405,15 +414,15 @@ }, "targets": [ { - "expr": "(node_memory_MemFree{instance=~\"$server\"} / node_memory_MemTotal{instance=~\"$server\"}) * 100", + "expr": "((node_memory_MemTotal{instance=~\"$server\"} - node_memory_MemFree{instance=~\"$server\"}) / node_memory_MemTotal{instance=~\"$server\"}) * 100", "intervalFactor": 2, "refId": "A", "step": 60, "target": "" } ], - "thresholds": "10, 20", - "title": "Free memory", + "thresholds": "80, 90", + "title": "Memory usage", "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ @@ -519,7 +528,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Disk usage", + "title": "Disk I/O", "tooltip": { "msResolution": false, "shared": true, @@ -554,9 +563,9 @@ "colorBackground": false, "colorValue": false, "colors": [ - "rgba(245, 54, 54, 0.9)", + "rgba(50, 172, 45, 0.97)", "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" + "rgba(245, 54, 54, 0.9)" ], "datasource": "${DS_PROMETHEUS}", "editable": true, @@ -607,15 +616,15 @@ }, "targets": [ { - "expr": "sum(node_filesystem_free{device!=\"rootfs\",instance=~\"$server\"}) / sum(node_filesystem_size{device!=\"rootfs\",instance=~\"$server\"})", + "expr": "(sum(node_filesystem_size{device!=\"rootfs\",instance=~\"$server\"}) - sum(node_filesystem_free{device!=\"rootfs\",instance=~\"$server\"})) / sum(node_filesystem_size{device!=\"rootfs\",instance=~\"$server\"})", "intervalFactor": 2, "refId": "A", "step": 60, "target": "" } ], - "thresholds": "0.10, 0.25", - "title": "Free disk space", + "thresholds": "0.75, 0.9", + "title": "Disk space usage", "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ @@ -890,4 +899,4 @@ } ], "overwrite": true -} \ No newline at end of file +} diff --git a/manifests/grafana/grafana-cm.yaml b/manifests/grafana/grafana-cm.yaml index db41fd60..fd512455 100644 --- a/manifests/grafana/grafana-cm.yaml +++ b/manifests/grafana/grafana-cm.yaml @@ -2109,7 +2109,7 @@ data: ], "overwrite": true } - node-dashboard.json: |- + node-dashboard.json: | { "dashboard": { "__inputs": [ @@ -2419,17 +2419,26 @@ data: { "expr": "node_memory_MemTotal{instance=~\"$server\"} - node_memory_MemFree{instance=~\"$server\"}", "intervalFactor": 2, - "legendFormat": "free memory", + "legendFormat": "memory usage", "metric": "memo", "refId": "A", "step": 4, "target": "" + }, + { + "expr": "node_memory_MemTotal{instance=~\"$server\"}", + "intervalFactor": 2, + "legendFormat": "memory total", + "metric": "memo", + "refId": "B", + "step": 4, + "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Free memory", + "title": "Memory usage", "tooltip": { "msResolution": false, "shared": true, @@ -2464,9 +2473,9 @@ data: "colorBackground": false, "colorValue": false, "colors": [ - "rgba(245, 54, 54, 0.9)", + "rgba(50, 172, 45, 0.97)", "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" + "rgba(245, 54, 54, 0.9)" ], "datasource": "${DS_PROMETHEUS}", "editable": true, @@ -2517,15 +2526,15 @@ data: }, "targets": [ { - "expr": "(node_memory_MemFree{instance=~\"$server\"} / node_memory_MemTotal{instance=~\"$server\"}) * 100", + "expr": "((node_memory_MemTotal{instance=~\"$server\"} - node_memory_MemFree{instance=~\"$server\"}) / node_memory_MemTotal{instance=~\"$server\"}) * 100", "intervalFactor": 2, "refId": "A", "step": 60, "target": "" } ], - "thresholds": "10, 20", - "title": "Free memory", + "thresholds": "80, 90", + "title": "Memory usage", "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ @@ -2631,7 +2640,7 @@ data: "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Disk usage", + "title": "Disk I/O", "tooltip": { "msResolution": false, "shared": true, @@ -2666,9 +2675,9 @@ data: "colorBackground": false, "colorValue": false, "colors": [ - "rgba(245, 54, 54, 0.9)", + "rgba(50, 172, 45, 0.97)", "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" + "rgba(245, 54, 54, 0.9)" ], "datasource": "${DS_PROMETHEUS}", "editable": true, @@ -2719,15 +2728,15 @@ data: }, "targets": [ { - "expr": "sum(node_filesystem_free{device!=\"rootfs\",instance=~\"$server\"}) / sum(node_filesystem_size{device!=\"rootfs\",instance=~\"$server\"})", + "expr": "(sum(node_filesystem_size{device!=\"rootfs\",instance=~\"$server\"}) - sum(node_filesystem_free{device!=\"rootfs\",instance=~\"$server\"})) / sum(node_filesystem_size{device!=\"rootfs\",instance=~\"$server\"})", "intervalFactor": 2, "refId": "A", "step": 60, "target": "" } ], - "thresholds": "0.10, 0.25", - "title": "Free disk space", + "thresholds": "0.75, 0.9", + "title": "Disk space usage", "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ From f5f00ec1bc2a3153162377dee417c626f893e83c Mon Sep 17 00:00:00 2001 From: Frederic Branczyk Date: Tue, 21 Feb 2017 14:26:07 +0100 Subject: [PATCH 2/3] prefer rate over irate --- assets/grafana/node-dashboard.json | 10 +++++----- manifests/grafana/grafana-cm.yaml | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/assets/grafana/node-dashboard.json b/assets/grafana/node-dashboard.json index 19b5ba2d..fb0c7645 100644 --- a/assets/grafana/node-dashboard.json +++ b/assets/grafana/node-dashboard.json @@ -502,7 +502,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (instance) (irate(node_disk_bytes_read{instance=~\"$server\"}[5m]))", + "expr": "sum by (instance) (rate(node_disk_bytes_read{instance=~\"$server\"}[2m]))", "hide": false, "intervalFactor": 4, "legendFormat": "read", @@ -511,14 +511,14 @@ "target": "" }, { - "expr": "sum by (instance) (irate(node_disk_bytes_written{instance=~\"$server\"}[5m]))", + "expr": "sum by (instance) (rate(node_disk_bytes_written{instance=~\"$server\"}[2m]))", "intervalFactor": 4, "legendFormat": "written", "refId": "B", "step": 8 }, { - "expr": "sum by (instance) (irate(node_disk_io_time_ms{instance=~\"$server\"}[5m]))", + "expr": "sum by (instance) (rate(node_disk_io_time_ms{instance=~\"$server\"}[2m]))", "intervalFactor": 4, "legendFormat": "io time", "refId": "C", @@ -696,7 +696,7 @@ "steppedLine": false, "targets": [ { - "expr": "irate(node_network_receive_bytes{instance=~\"$server\",device!~\"lo\"}[5m])", + "expr": "rate(node_network_receive_bytes{instance=~\"$server\",device!~\"lo\"}[5m])", "hide": false, "intervalFactor": 2, "legendFormat": "{{device}}", @@ -790,7 +790,7 @@ "steppedLine": false, "targets": [ { - "expr": "irate(node_network_transmit_bytes{instance=~\"$server\",device!~\"lo\"}[5m])", + "expr": "rate(node_network_transmit_bytes{instance=~\"$server\",device!~\"lo\"}[5m])", "hide": false, "intervalFactor": 2, "legendFormat": "{{device}}", diff --git a/manifests/grafana/grafana-cm.yaml b/manifests/grafana/grafana-cm.yaml index fd512455..081bb2f6 100644 --- a/manifests/grafana/grafana-cm.yaml +++ b/manifests/grafana/grafana-cm.yaml @@ -2614,7 +2614,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum by (instance) (irate(node_disk_bytes_read{instance=~\"$server\"}[5m]))", + "expr": "sum by (instance) (rate(node_disk_bytes_read{instance=~\"$server\"}[2m]))", "hide": false, "intervalFactor": 4, "legendFormat": "read", @@ -2623,14 +2623,14 @@ data: "target": "" }, { - "expr": "sum by (instance) (irate(node_disk_bytes_written{instance=~\"$server\"}[5m]))", + "expr": "sum by (instance) (rate(node_disk_bytes_written{instance=~\"$server\"}[2m]))", "intervalFactor": 4, "legendFormat": "written", "refId": "B", "step": 8 }, { - "expr": "sum by (instance) (irate(node_disk_io_time_ms{instance=~\"$server\"}[5m]))", + "expr": "sum by (instance) (rate(node_disk_io_time_ms{instance=~\"$server\"}[2m]))", "intervalFactor": 4, "legendFormat": "io time", "refId": "C", @@ -2808,7 +2808,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "irate(node_network_receive_bytes{instance=~\"$server\",device!~\"lo\"}[5m])", + "expr": "rate(node_network_receive_bytes{instance=~\"$server\",device!~\"lo\"}[5m])", "hide": false, "intervalFactor": 2, "legendFormat": "{{device}}", @@ -2902,7 +2902,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "irate(node_network_transmit_bytes{instance=~\"$server\",device!~\"lo\"}[5m])", + "expr": "rate(node_network_transmit_bytes{instance=~\"$server\",device!~\"lo\"}[5m])", "hide": false, "intervalFactor": 2, "legendFormat": "{{device}}", From 615f45e605e5edd781a2273cf6463d08bba510e1 Mon Sep 17 00:00:00 2001 From: Frederic Branczyk Date: Tue, 21 Feb 2017 14:59:58 +0100 Subject: [PATCH 3/3] prefer `=` over `=~` --- assets/grafana/node-dashboard.json | 26 +++++++++++++------------- manifests/grafana/grafana-cm.yaml | 26 +++++++++++++------------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/assets/grafana/node-dashboard.json b/assets/grafana/node-dashboard.json index fb0c7645..272d54ea 100644 --- a/assets/grafana/node-dashboard.json +++ b/assets/grafana/node-dashboard.json @@ -100,7 +100,7 @@ "steppedLine": false, "targets": [ { - "expr": "100 - (avg by (cpu) (irate(node_cpu{mode=\"idle\", instance=~\"$server\"}[5m])) * 100)", + "expr": "100 - (avg by (cpu) (irate(node_cpu{mode=\"idle\", instance=\"$server\"}[5m])) * 100)", "hide": false, "intervalFactor": 10, "legendFormat": "{{cpu}}", @@ -188,7 +188,7 @@ "steppedLine": false, "targets": [ { - "expr": "node_load1{instance=~\"$server\"}", + "expr": "node_load1{instance=\"$server\"}", "intervalFactor": 4, "legendFormat": "load 1m", "refId": "A", @@ -196,7 +196,7 @@ "target": "" }, { - "expr": "node_load5{instance=~\"$server\"}", + "expr": "node_load5{instance=\"$server\"}", "intervalFactor": 4, "legendFormat": "load 5m", "refId": "B", @@ -204,7 +204,7 @@ "target": "" }, { - "expr": "node_load15{instance=~\"$server\"}", + "expr": "node_load15{instance=\"$server\"}", "intervalFactor": 4, "legendFormat": "load 15m", "refId": "C", @@ -305,7 +305,7 @@ "steppedLine": false, "targets": [ { - "expr": "node_memory_MemTotal{instance=~\"$server\"} - node_memory_MemFree{instance=~\"$server\"}", + "expr": "node_memory_MemTotal{instance=\"$server\"} - node_memory_MemFree{instance=\"$server\"}", "intervalFactor": 2, "legendFormat": "memory usage", "metric": "memo", @@ -314,7 +314,7 @@ "target": "" }, { - "expr": "node_memory_MemTotal{instance=~\"$server\"}", + "expr": "node_memory_MemTotal{instance=\"$server\"}", "intervalFactor": 2, "legendFormat": "memory total", "metric": "memo", @@ -414,7 +414,7 @@ }, "targets": [ { - "expr": "((node_memory_MemTotal{instance=~\"$server\"} - node_memory_MemFree{instance=~\"$server\"}) / node_memory_MemTotal{instance=~\"$server\"}) * 100", + "expr": "((node_memory_MemTotal{instance=\"$server\"} - node_memory_MemFree{instance=\"$server\"}) / node_memory_MemTotal{instance=\"$server\"}) * 100", "intervalFactor": 2, "refId": "A", "step": 60, @@ -502,7 +502,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (instance) (rate(node_disk_bytes_read{instance=~\"$server\"}[2m]))", + "expr": "sum by (instance) (rate(node_disk_bytes_read{instance=\"$server\"}[2m]))", "hide": false, "intervalFactor": 4, "legendFormat": "read", @@ -511,14 +511,14 @@ "target": "" }, { - "expr": "sum by (instance) (rate(node_disk_bytes_written{instance=~\"$server\"}[2m]))", + "expr": "sum by (instance) (rate(node_disk_bytes_written{instance=\"$server\"}[2m]))", "intervalFactor": 4, "legendFormat": "written", "refId": "B", "step": 8 }, { - "expr": "sum by (instance) (rate(node_disk_io_time_ms{instance=~\"$server\"}[2m]))", + "expr": "sum by (instance) (rate(node_disk_io_time_ms{instance=\"$server\"}[2m]))", "intervalFactor": 4, "legendFormat": "io time", "refId": "C", @@ -616,7 +616,7 @@ }, "targets": [ { - "expr": "(sum(node_filesystem_size{device!=\"rootfs\",instance=~\"$server\"}) - sum(node_filesystem_free{device!=\"rootfs\",instance=~\"$server\"})) / sum(node_filesystem_size{device!=\"rootfs\",instance=~\"$server\"})", + "expr": "(sum(node_filesystem_size{device!=\"rootfs\",instance=\"$server\"}) - sum(node_filesystem_free{device!=\"rootfs\",instance=\"$server\"})) / sum(node_filesystem_size{device!=\"rootfs\",instance=\"$server\"})", "intervalFactor": 2, "refId": "A", "step": 60, @@ -696,7 +696,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(node_network_receive_bytes{instance=~\"$server\",device!~\"lo\"}[5m])", + "expr": "rate(node_network_receive_bytes{instance=\"$server\",device!~\"lo\"}[5m])", "hide": false, "intervalFactor": 2, "legendFormat": "{{device}}", @@ -790,7 +790,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(node_network_transmit_bytes{instance=~\"$server\",device!~\"lo\"}[5m])", + "expr": "rate(node_network_transmit_bytes{instance=\"$server\",device!~\"lo\"}[5m])", "hide": false, "intervalFactor": 2, "legendFormat": "{{device}}", diff --git a/manifests/grafana/grafana-cm.yaml b/manifests/grafana/grafana-cm.yaml index 081bb2f6..ff418fec 100644 --- a/manifests/grafana/grafana-cm.yaml +++ b/manifests/grafana/grafana-cm.yaml @@ -2212,7 +2212,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "100 - (avg by (cpu) (irate(node_cpu{mode=\"idle\", instance=~\"$server\"}[5m])) * 100)", + "expr": "100 - (avg by (cpu) (irate(node_cpu{mode=\"idle\", instance=\"$server\"}[5m])) * 100)", "hide": false, "intervalFactor": 10, "legendFormat": "{{cpu}}", @@ -2300,7 +2300,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "node_load1{instance=~\"$server\"}", + "expr": "node_load1{instance=\"$server\"}", "intervalFactor": 4, "legendFormat": "load 1m", "refId": "A", @@ -2308,7 +2308,7 @@ data: "target": "" }, { - "expr": "node_load5{instance=~\"$server\"}", + "expr": "node_load5{instance=\"$server\"}", "intervalFactor": 4, "legendFormat": "load 5m", "refId": "B", @@ -2316,7 +2316,7 @@ data: "target": "" }, { - "expr": "node_load15{instance=~\"$server\"}", + "expr": "node_load15{instance=\"$server\"}", "intervalFactor": 4, "legendFormat": "load 15m", "refId": "C", @@ -2417,7 +2417,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "node_memory_MemTotal{instance=~\"$server\"} - node_memory_MemFree{instance=~\"$server\"}", + "expr": "node_memory_MemTotal{instance=\"$server\"} - node_memory_MemFree{instance=\"$server\"}", "intervalFactor": 2, "legendFormat": "memory usage", "metric": "memo", @@ -2426,7 +2426,7 @@ data: "target": "" }, { - "expr": "node_memory_MemTotal{instance=~\"$server\"}", + "expr": "node_memory_MemTotal{instance=\"$server\"}", "intervalFactor": 2, "legendFormat": "memory total", "metric": "memo", @@ -2526,7 +2526,7 @@ data: }, "targets": [ { - "expr": "((node_memory_MemTotal{instance=~\"$server\"} - node_memory_MemFree{instance=~\"$server\"}) / node_memory_MemTotal{instance=~\"$server\"}) * 100", + "expr": "((node_memory_MemTotal{instance=\"$server\"} - node_memory_MemFree{instance=\"$server\"}) / node_memory_MemTotal{instance=\"$server\"}) * 100", "intervalFactor": 2, "refId": "A", "step": 60, @@ -2614,7 +2614,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum by (instance) (rate(node_disk_bytes_read{instance=~\"$server\"}[2m]))", + "expr": "sum by (instance) (rate(node_disk_bytes_read{instance=\"$server\"}[2m]))", "hide": false, "intervalFactor": 4, "legendFormat": "read", @@ -2623,14 +2623,14 @@ data: "target": "" }, { - "expr": "sum by (instance) (rate(node_disk_bytes_written{instance=~\"$server\"}[2m]))", + "expr": "sum by (instance) (rate(node_disk_bytes_written{instance=\"$server\"}[2m]))", "intervalFactor": 4, "legendFormat": "written", "refId": "B", "step": 8 }, { - "expr": "sum by (instance) (rate(node_disk_io_time_ms{instance=~\"$server\"}[2m]))", + "expr": "sum by (instance) (rate(node_disk_io_time_ms{instance=\"$server\"}[2m]))", "intervalFactor": 4, "legendFormat": "io time", "refId": "C", @@ -2728,7 +2728,7 @@ data: }, "targets": [ { - "expr": "(sum(node_filesystem_size{device!=\"rootfs\",instance=~\"$server\"}) - sum(node_filesystem_free{device!=\"rootfs\",instance=~\"$server\"})) / sum(node_filesystem_size{device!=\"rootfs\",instance=~\"$server\"})", + "expr": "(sum(node_filesystem_size{device!=\"rootfs\",instance=\"$server\"}) - sum(node_filesystem_free{device!=\"rootfs\",instance=\"$server\"})) / sum(node_filesystem_size{device!=\"rootfs\",instance=\"$server\"})", "intervalFactor": 2, "refId": "A", "step": 60, @@ -2808,7 +2808,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(node_network_receive_bytes{instance=~\"$server\",device!~\"lo\"}[5m])", + "expr": "rate(node_network_receive_bytes{instance=\"$server\",device!~\"lo\"}[5m])", "hide": false, "intervalFactor": 2, "legendFormat": "{{device}}", @@ -2902,7 +2902,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(node_network_transmit_bytes{instance=~\"$server\",device!~\"lo\"}[5m])", + "expr": "rate(node_network_transmit_bytes{instance=\"$server\",device!~\"lo\"}[5m])", "hide": false, "intervalFactor": 2, "legendFormat": "{{device}}",