diff --git a/jsonnet/kube-prometheus/alerts/prometheus-operator.libsonnet b/jsonnet/kube-prometheus/alerts/prometheus-operator.libsonnet index a430c505..580aa866 100644 --- a/jsonnet/kube-prometheus/alerts/prometheus-operator.libsonnet +++ b/jsonnet/kube-prometheus/alerts/prometheus-operator.libsonnet @@ -4,6 +4,19 @@ { name: 'prometheus-operator', rules: [ + { + alert: 'PrometheusOperatorWatchErrors', + expr: ||| + (sum by (controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{%(prometheusOperatorSelector)s}[1h])) / sum by (controller,namespace) (rate(prometheus_operator_watch_operations_total{%(prometheusOperatorSelector)s}[1h]))) > 0.1 + ||| % $._config, + labels: { + severity: 'warning', + }, + annotations: { + message: 'Errors while performing watch operations in controller {{$labels.controller}} in {{$labels.namespace}} namespace.', + }, + 'for': '15m', + }, { alert: 'PrometheusOperatorReconcileErrors', expr: ||| diff --git a/manifests/prometheus-rules.yaml b/manifests/prometheus-rules.yaml index aefacdf1..94c3fe4e 100644 --- a/manifests/prometheus-rules.yaml +++ b/manifests/prometheus-rules.yaml @@ -1793,6 +1793,15 @@ spec: severity: warning - name: prometheus-operator rules: + - alert: PrometheusOperatorWatchErrors + annotations: + message: Errors while performing watch operations in controller {{$labels.controller}} + in {{$labels.namespace}} namespace. + expr: | + (sum by (controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{job="prometheus-operator",namespace="monitoring"}[1h])) / sum by (controller,namespace) (rate(prometheus_operator_watch_operations_total{job="prometheus-operator",namespace="monitoring"}[1h]))) > 0.1 + for: 15m + labels: + severity: warning - alert: PrometheusOperatorReconcileErrors annotations: message: Errors while reconciling {{ $labels.controller }} in {{ $labels.namespace