mirrors/kube-prometheus
1
0
Fork 0
mirror of https://github.com/prometheus-operator/kube-prometheus.git synced 2025-11-04 01:51:01 +01:00
Code Issues Projects Releases Wiki Activity

Merge pull request #610 from lilic/add-more-alerts

Browse Source 
Add PrometheusOperatorListErrors and fix PrometheusOperatorWatchErrors threshold
This commit is contained in:
Frederic Branczyk 2020-07-15 13:19:45 +02:00 committed by GitHub
commit b55c2825f7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 28 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
jsonnet/kube-prometheus/alerts/prometheus-operator.libsonnet
View File

@ -5,15 +5,28 @@
name: 'prometheus-operator', name: 'prometheus-operator',
rules: [ rules: [
{ {
alert: 'PrometheusOperatorWatchErrors', alert: 'PrometheusOperatorListErrors',
expr: ||| expr: |||
(sum by (controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{%(prometheusOperatorSelector)s}[1h])) / sum by (controller,namespace) (rate(prometheus_operator_watch_operations_total{%(prometheusOperatorSelector)s}[1h]))) > 0.1 (sum by (controller,namespace) (rate(prometheus_operator_list_operations_failed_total{%(prometheusOperatorSelector)s}[1h])) / sum by (controller,namespace) (rate(prometheus_operator_list_operations_total{%(prometheusOperatorSelector)s}[1h]))) > 0.4
||| % $._config, ||| % $._config,
labels: { labels: {
severity: 'warning', severity: 'warning',
}, },
annotations: { annotations: {
message: 'Errors while performing watch operations in controller {{$labels.controller}} in {{$labels.namespace}} namespace.', message: 'Errors while performing List operations in controller {{$labels.controller}} in {{$labels.namespace}} namespace.',
},
'for': '15m',
},
{
alert: 'PrometheusOperatorWatchErrors',
expr: |||
(sum by (controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{%(prometheusOperatorSelector)s}[1h])) / sum by (controller,namespace) (rate(prometheus_operator_watch_operations_total{%(prometheusOperatorSelector)s}[1h]))) > 0.4
||| % $._config,
labels: {
severity: 'warning',
},
annotations: {
message: 'Errors while performing Watch operations in controller {{$labels.controller}} in {{$labels.namespace}} namespace.',
}, },
'for': '15m', 'for': '15m',
}, },

15
manifests/prometheus-rules.yaml
View File

@ -1793,12 +1793,21 @@ spec:
severity: warning severity: warning
- name: prometheus-operator - name: prometheus-operator
rules: rules:
- alert: PrometheusOperatorWatchErrors - alert: PrometheusOperatorListErrors
annotations: annotations:
message: Errors while performing watch operations in controller {{$labels.controller}} message: Errors while performing List operations in controller {{$labels.controller}}
in {{$labels.namespace}} namespace. in {{$labels.namespace}} namespace.
expr: | expr: |
(sum by (controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{job="prometheus-operator",namespace="monitoring"}[1h])) / sum by (controller,namespace) (rate(prometheus_operator_watch_operations_total{job="prometheus-operator",namespace="monitoring"}[1h]))) > 0.1 (sum by (controller,namespace) (rate(prometheus_operator_list_operations_failed_total{job="prometheus-operator",namespace="monitoring"}[1h])) / sum by (controller,namespace) (rate(prometheus_operator_list_operations_total{job="prometheus-operator",namespace="monitoring"}[1h]))) > 0.4
for: 15m
labels:
severity: warning
- alert: PrometheusOperatorWatchErrors
annotations:
message: Errors while performing Watch operations in controller {{$labels.controller}}
in {{$labels.namespace}} namespace.
expr: |
(sum by (controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{job="prometheus-operator",namespace="monitoring"}[1h])) / sum by (controller,namespace) (rate(prometheus_operator_watch_operations_total{job="prometheus-operator",namespace="monitoring"}[1h]))) > 0.4
for: 15m for: 15m
labels: labels:
severity: warning severity: warning