Merge pull request #1231 from dgrisonnet/fix-adapter-queries

Consolidate intervals used in prometheus-adapter CPU queries

jsonnet/kube-prometheus/components/prometheus-adapter.libsonnet

@@ -22,13 +22,40 @@ local defaults = {
     for labelName in std.objectFields(defaults.commonLabels)
     if !std.setMember(labelName, ['app.kubernetes.io/version'])
   },
+  // Default range intervals are equal to 4 times the default scrape interval.
+  // This is done in order to follow Prometheus rule of thumb with irate().
+  rangeIntervals: {
+    kubelet: '4m',
+    nodeExporter: '4m',
+    windowsExporter: '4m',
+  },
 
   prometheusURL: error 'must provide prometheusURL',
   config: {
     resourceRules: {
       cpu: {
-        containerQuery: 'sum(irate(container_cpu_usage_seconds_total{<<.LabelMatchers>>,container!="",pod!=""}[5m])) by (<<.GroupBy>>)',
+        containerQuery: |||
-        nodeQuery: 'sum(1 - irate(node_cpu_seconds_total{mode="idle"}[5m]) * on(namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:{<<.LabelMatchers>>}) by (<<.GroupBy>>) or sum (1- irate(windows_cpu_time_total{mode="idle", job="windows-exporter",<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>)',
+          sum by (<<.GroupBy>>) (
+            irate (
+                container_cpu_usage_seconds_total{<<.LabelMatchers>>,container!="",pod!=""}[%(kubelet)s]
+            )
+          )
+        ||| % $.rangeIntervals,
+        nodeQuery: |||
+          sum by (<<.GroupBy>>) (
+            1 - irate(
+              node_cpu_seconds_total{mode="idle"}[%(nodeExporter)s]
+            )
+            * on(namespace, pod) group_left(node) (
+              node_namespace_pod:kube_pod_info:{<<.LabelMatchers>>}
+            )
+          )
+          or sum by (<<.GroupBy>>) (
+            1 - irate(
+              windows_cpu_time_total{mode="idle", job="windows-exporter",<<.LabelMatchers>>}[%(windowsExporter)s]
+            )
+          )
+        ||| % $.rangeIntervals,
         resources: {
           overrides: {
             node: { resource: 'node' },
@@ -39,8 +66,23 @@ local defaults = {
         containerLabel: 'container',
       },
       memory: {
-        containerQuery: 'sum(container_memory_working_set_bytes{<<.LabelMatchers>>,container!="",pod!=""}) by (<<.GroupBy>>)',
+        containerQuery: |||
-        nodeQuery: 'sum(node_memory_MemTotal_bytes{job="node-exporter",<<.LabelMatchers>>} - node_memory_MemAvailable_bytes{job="node-exporter",<<.LabelMatchers>>}) by (<<.GroupBy>>) or sum(windows_cs_physical_memory_bytes{job="windows-exporter",<<.LabelMatchers>>} - windows_memory_available_bytes{job="windows-exporter",<<.LabelMatchers>>}) by (<<.GroupBy>>)',
+          sum by (<<.GroupBy>>) (
+            container_memory_working_set_bytes{<<.LabelMatchers>>,container!="",pod!=""}
+          )
+        |||,
+        nodeQuery: |||
+          sum by (<<.GroupBy>>) (
+            node_memory_MemTotal_bytes{job="node-exporter",<<.LabelMatchers>>}
+            -
+            node_memory_MemAvailable_bytes{job="node-exporter",<<.LabelMatchers>>}
+          )
+          or sum by (<<.GroupBy>>) (
+            windows_cs_physical_memory_bytes{job="windows-exporter",<<.LabelMatchers>>}
+            -
+            windows_memory_available_bytes{job="windows-exporter",<<.LabelMatchers>>}
+          )
+        |||,
         resources: {
           overrides: {
             instance: { resource: 'node' },

jsonnet/kube-prometheus/lib/utils.libsonnet

@@ -0,0 +1,7 @@
+{
+  // rangeInterval takes a scrape interval and convert its to a range interval
+  // following Prometheus rule of thumb for rate() and irate().
+  rangeInterval(i='1m'):
+    local interval = std.parseInt(std.substr(i, 0, std.length(i) - 1));
+    interval * 4 + i[std.length(i) - 1],
+}

jsonnet/kube-prometheus/main.libsonnet

@@ -11,6 +11,8 @@ local prometheus = import './components/prometheus.libsonnet';
 
 local platformPatch = import './platforms/platforms.libsonnet';
 
+local utils = import './lib/utils.libsonnet';
+
 {
   // using `values` as this is similar to helm
   values:: {
@@ -97,6 +99,10 @@ local platformPatch = import './platforms/platforms.libsonnet';
       version: $.values.common.versions.prometheusAdapter,
       image: $.values.common.images.prometheusAdapter,
       prometheusURL: 'http://prometheus-' + $.values.prometheus.name + '.' + $.values.common.namespace + '.svc.cluster.local:9090/',
+      rangeIntervals+: {
+        kubelet: utils.rangeInterval($.kubernetesControlPlane.serviceMonitorKubelet.spec.endpoints[0].interval),
+        nodeExporter: utils.rangeInterval($.nodeExporter.serviceMonitor.spec.endpoints[0].interval),
+      },
     },
     prometheusOperator: {
       namespace: $.values.common.namespace,

manifests/prometheus-adapter-configMap.yaml

@@ -4,8 +4,26 @@ data:
     "resourceRules":
       "cpu":
         "containerLabel": "container"
-        "containerQuery": "sum(irate(container_cpu_usage_seconds_total{<<.LabelMatchers>>,container!=\"\",pod!=\"\"}[5m])) by (<<.GroupBy>>)"
+        "containerQuery": |
-        "nodeQuery": "sum(1 - irate(node_cpu_seconds_total{mode=\"idle\"}[5m]) * on(namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:{<<.LabelMatchers>>}) by (<<.GroupBy>>) or sum (1- irate(windows_cpu_time_total{mode=\"idle\", job=\"windows-exporter\",<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>)"
+          sum by (<<.GroupBy>>) (
+            irate (
+                container_cpu_usage_seconds_total{<<.LabelMatchers>>,container!="",pod!=""}[120s]
+            )
+          )
+        "nodeQuery": |
+          sum by (<<.GroupBy>>) (
+            1 - irate(
+              node_cpu_seconds_total{mode="idle"}[60s]
+            )
+            * on(namespace, pod) group_left(node) (
+              node_namespace_pod:kube_pod_info:{<<.LabelMatchers>>}
+            )
+          )
+          or sum by (<<.GroupBy>>) (
+            1 - irate(
+              windows_cpu_time_total{mode="idle", job="windows-exporter",<<.LabelMatchers>>}[4m]
+            )
+          )
         "resources":
           "overrides":
             "namespace":
@@ -16,8 +34,21 @@ data:
               "resource": "pod"
       "memory":
         "containerLabel": "container"
-        "containerQuery": "sum(container_memory_working_set_bytes{<<.LabelMatchers>>,container!=\"\",pod!=\"\"}) by (<<.GroupBy>>)"
+        "containerQuery": |
-        "nodeQuery": "sum(node_memory_MemTotal_bytes{job=\"node-exporter\",<<.LabelMatchers>>} - node_memory_MemAvailable_bytes{job=\"node-exporter\",<<.LabelMatchers>>}) by (<<.GroupBy>>) or sum(windows_cs_physical_memory_bytes{job=\"windows-exporter\",<<.LabelMatchers>>} - windows_memory_available_bytes{job=\"windows-exporter\",<<.LabelMatchers>>}) by (<<.GroupBy>>)"
+          sum by (<<.GroupBy>>) (
+            container_memory_working_set_bytes{<<.LabelMatchers>>,container!="",pod!=""}
+          )
+        "nodeQuery": |
+          sum by (<<.GroupBy>>) (
+            node_memory_MemTotal_bytes{job="node-exporter",<<.LabelMatchers>>}
+            -
+            node_memory_MemAvailable_bytes{job="node-exporter",<<.LabelMatchers>>}
+          )
+          or sum by (<<.GroupBy>>) (
+            windows_cs_physical_memory_bytes{job="windows-exporter",<<.LabelMatchers>>}
+            -
+            windows_memory_available_bytes{job="windows-exporter",<<.LabelMatchers>>}
+          )
         "resources":
           "overrides":
             "instance":