Merge pull request #1253 from ndegory/update-doc-for-0.8

update doc on Prometheus rule updates since release 0.8

docs/developing-prometheus-rules-and-grafana-dashboards.md

@@ -219,72 +219,113 @@ local kp = (import 'kube-prometheus/main.libsonnet') + {
 ```
 ### Changing default rules
 
-Along with adding additional rules, we give the user the option to filter or adjust the existing rules imported by `kube-prometheus/kube-prometheus.libsonnet`. The recording rules can be found in [kube-prometheus/rules](../jsonnet/kube-prometheus/rules) and [kubernetes-mixin/rules](https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/rules) while the alerting rules can be found in [kube-prometheus/alerts](../jsonnet/kube-prometheus/alerts) and [kubernetes-mixin/alerts](https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/alerts).
+Along with adding additional rules, we give the user the option to filter or adjust the existing rules imported by `kube-prometheus/main.libsonnet`. The recording rules can be found in [kube-prometheus/components/mixin/rules](../jsonnet/kube-prometheus/components/mixin/rules) and [kubernetes-mixin/rules](https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/rules) while the alerting rules can be found in [kube-prometheus/components/mixin/alerts](../jsonnet/kube-prometheus/components/mixin/alerts) and [kubernetes-mixin/alerts](https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/alerts).
 
 Knowing which rules to change, the user can now use functions from the [Jsonnet standard library](https://jsonnet.org/ref/stdlib.html) to make these changes. Below are examples of both a filter and an adjustment being made to the default rules. These changes can be assigned to a local variable and then added to the `local kp` object as seen in the examples above.
 
 #### Filter
-Here the alert `KubeStatefulSetReplicasMismatch` is being filtered out of the group `kubernetes-apps`. The default rule can be seen [here](https://github.com/kubernetes-monitoring/kubernetes-mixin/blob/master/alerts/apps_alerts.libsonnet).
+Here the alert `KubeStatefulSetReplicasMismatch` is being filtered out of the group `kubernetes-apps`. The default rule can be seen [here](https://github.com/kubernetes-monitoring/kubernetes-mixin/blob/master/alerts/apps_alerts.libsonnet). You first need to find out in which component the rule is defined (here it is kuberentesControlPlane).
 ```jsonnet
 local filter = {
-  prometheusAlerts+:: {
+  kubernetesControlPlane+: {
-    groups: std.map(
+    prometheusRule+:: {
-      function(group)
+      spec+: {
-        if group.name == 'kubernetes-apps' then
+        groups: std.map(
-          group {
+          function(group)
-            rules: std.filter(function(rule)
+            if group.name == 'kubernetes-apps' then
-              rule.alert != "KubeStatefulSetReplicasMismatch",
+              group {
-              group.rules
+                rules: std.filter(
-            )
+                  function(rule)
-          }
+                    rule.alert != 'KubeStatefulSetReplicasMismatch',
-        else
+                  group.rules
-          group,
+                ),
-      super.groups
+              }
-    ),
+            else
+              group,
+          super.groups
+        ),
+      },
+    },
   },
 };
 ```
+
 #### Adjustment
-Here the expression for the alert used above is updated from its previous value. The default rule can be seen [here](https://github.com/kubernetes-monitoring/kubernetes-mixin/blob/master/alerts/apps_alerts.libsonnet).
+Here the expression for another alert in the same component is updated from its previous value. The default rule can be seen [here](https://github.com/kubernetes-monitoring/kubernetes-mixin/blob/master/alerts/apps_alerts.libsonnet).
 ```jsonnet
 local update = {
-  prometheusAlerts+:: {
+  kubernetesControlPlane+: {
-    groups: std.map(
+    prometheusRule+:: {
-      function(group)
+      spec+: {
-        if group.name == 'kubernetes-apps' then
+        groups: std.map(
-          group {
+          function(group)
-            rules: std.map(
+            if group.name == 'kubernetes-apps' then
-              function(rule)
+              group {
-                if rule.alert == "KubeStatefulSetReplicasMismatch" then
+                rules: std.map(
-                  rule {
+                  function(rule)
-                    expr: "kube_statefulset_status_replicas_ready{job=\"kube-state-metrics\",statefulset!=\"vault\"} != kube_statefulset_status_replicas{job=\"kube-state-metrics\",statefulset!=\"vault\"}"
+                    if rule.alert == 'KubePodCrashLooping' then
-                  }
+                      rule {
-                else
+                        expr: 'rate(kube_pod_container_status_restarts_total{namespace=kube-system,job="kube-state-metrics"}[10m]) * 60 * 5 > 0',
-                  rule,
+                      }
-                group.rules
+                    else
-            )
+                      rule,
-          }
+                  group.rules
-        else
+                ),
-          group,
+              }
-      super.groups
+            else
-    ),
+              group,
+          super.groups
+        ),
+      },
+    },
   },
 };
 ```
+
 Using the example from above about adding in pre-rendered rules, the new local variables can be added in as follows:
 ```jsonnet
-local kp = (import 'kube-prometheus/kube-prometheus.libsonnet') + filter + update + {
+local add = {
-    prometheusAlerts+:: (import 'existingrule.json'),
+  exampleApplication:: {
+    prometheusRule+: {
+      apiVersion: 'monitoring.coreos.com/v1',
+      kind: 'PrometheusRule',
+      metadata: {
+        name: 'example-application-rules',
+        namespace: $.values.common.namespace,
+      },
+      spec: (import 'existingrule.json'),
+    },
+  },
 };
-
+local kp = (import 'kube-prometheus/main.libsonnet') + filter + update + add;
-{ ['00namespace-' + name]: kp.kubePrometheus[name] for name in std.objectFields(kp.kubePrometheus) } +
+local kp = (import 'kube-prometheus/main.libsonnet') +
-{ ['0prometheus-operator-' + name]: kp.prometheusOperator[name] for name in std.objectFields(kp.prometheusOperator) } +
+            filter +
+            update +
+            add + {
+	      values+:: {
+                common+: {
+                  namespace: 'monitoring',
+                },
+              },
+            };
+{ 'setup/0namespace-namespace': kp.kubePrometheus.namespace } +
+{
+  ['setup/prometheus-operator-' + name]: kp.prometheusOperator[name]
+  for name in std.filter((function(name) name != 'serviceMonitor' && name != 'prometheusRule'), std.objectFields(kp.prometheusOperator))
+} +
+// serviceMonitor and prometheusRule are separated so that they can be created after the CRDs are ready
+{ 'prometheus-operator-serviceMonitor': kp.prometheusOperator.serviceMonitor } +
+{ 'prometheus-operator-prometheusRule': kp.prometheusOperator.prometheusRule } +
+{ 'kube-prometheus-prometheusRule': kp.kubePrometheus.prometheusRule } +
 { ['node-exporter-' + name]: kp.nodeExporter[name] for name in std.objectFields(kp.nodeExporter) } +
+{ ['blackbox-exporter-' + name]: kp.blackboxExporter[name] for name in std.objectFields(kp.blackboxExporter) } +
 { ['kube-state-metrics-' + name]: kp.kubeStateMetrics[name] for name in std.objectFields(kp.kubeStateMetrics) } +
 { ['alertmanager-' + name]: kp.alertmanager[name] for name in std.objectFields(kp.alertmanager) } +
 { ['prometheus-' + name]: kp.prometheus[name] for name in std.objectFields(kp.prometheus) } +
 { ['prometheus-adapter-' + name]: kp.prometheusAdapter[name] for name in std.objectFields(kp.prometheusAdapter) } +
-{ ['grafana-' + name]: kp.grafana[name] for name in std.objectFields(kp.grafana) }
+{ ['grafana-' + name]: kp.grafana[name] for name in std.objectFields(kp.grafana) } +
+{ ['kubernetes-' + name]: kp.kubernetesControlPlane[name] for name in std.objectFields(kp.kubernetesControlPlane) } +
+{ ['exampleApplication-' + name]: kp.exampleApplication[name] for name in std.objectFields(kp.exampleApplication) }
 ```
 ## Dashboards
 

examples/changing-default-rules.libsonnet

@@ -0,0 +1,92 @@
+local filter = {
+  kubernetesControlPlane+: {
+    prometheusRule+:: {
+      spec+: {
+        groups: std.map(
+          function(group)
+            if group.name == 'kubernetes-apps' then
+              group {
+                rules: std.filter(
+                  function(rule)
+                    rule.alert != 'KubeStatefulSetReplicasMismatch',
+                  group.rules
+                ),
+              }
+            else
+              group,
+          super.groups
+        ),
+      },
+    },
+  },
+};
+local update = {
+  kubernetesControlPlane+: {
+    prometheusRule+:: {
+      spec+: {
+        groups: std.map(
+          function(group)
+            if group.name == 'kubernetes-apps' then
+              group {
+                rules: std.map(
+                  function(rule)
+                    if rule.alert == 'KubePodCrashLooping' then
+                      rule {
+                        expr: 'rate(kube_pod_container_status_restarts_total{namespace=kube-system,job="kube-state-metrics"}[10m]) * 60 * 5 > 0',
+                      }
+                    else
+                      rule,
+                  group.rules
+                ),
+              }
+            else
+              group,
+          super.groups
+        ),
+      },
+    },
+  },
+};
+
+local add = {
+  exampleApplication:: {
+    prometheusRule+: {
+      apiVersion: 'monitoring.coreos.com/v1',
+      kind: 'PrometheusRule',
+      metadata: {
+        name: 'example-application-rules',
+        namespace: $.values.common.namespace,
+      },
+      spec: (import 'existingrule.json'),
+    },
+  },
+};
+local kp = (import 'kube-prometheus/main.libsonnet') +
+           filter +
+           update +
+           add + {
+  values+:: {
+    common+: {
+      namespace: 'monitoring',
+    },
+  },
+};
+
+{ 'setup/0namespace-namespace': kp.kubePrometheus.namespace } +
+{
+  ['setup/prometheus-operator-' + name]: kp.prometheusOperator[name]
+  for name in std.filter((function(name) name != 'serviceMonitor' && name != 'prometheusRule'), std.objectFields(kp.prometheusOperator))
+} +
+// serviceMonitor and prometheusRule are separated so that they can be created after the CRDs are ready
+{ 'prometheus-operator-serviceMonitor': kp.prometheusOperator.serviceMonitor } +
+{ 'prometheus-operator-prometheusRule': kp.prometheusOperator.prometheusRule } +
+{ 'kube-prometheus-prometheusRule': kp.kubePrometheus.prometheusRule } +
+{ ['node-exporter-' + name]: kp.nodeExporter[name] for name in std.objectFields(kp.nodeExporter) } +
+{ ['blackbox-exporter-' + name]: kp.blackboxExporter[name] for name in std.objectFields(kp.blackboxExporter) } +
+{ ['kube-state-metrics-' + name]: kp.kubeStateMetrics[name] for name in std.objectFields(kp.kubeStateMetrics) } +
+{ ['alertmanager-' + name]: kp.alertmanager[name] for name in std.objectFields(kp.alertmanager) } +
+{ ['prometheus-' + name]: kp.prometheus[name] for name in std.objectFields(kp.prometheus) } +
+{ ['prometheus-adapter-' + name]: kp.prometheusAdapter[name] for name in std.objectFields(kp.prometheusAdapter) } +
+{ ['grafana-' + name]: kp.grafana[name] for name in std.objectFields(kp.grafana) } +
+{ ['kubernetes-' + name]: kp.kubernetesControlPlane[name] for name in std.objectFields(kp.kubernetesControlPlane) } +
+{ ['exampleApplication-' + name]: kp.exampleApplication[name] for name in std.objectFields(kp.exampleApplication) }