From 5fa86178a6cd5532da4d55cea42bedd73e949eb2 Mon Sep 17 00:00:00 2001
From: Douglas Camata <159076+douglascamata@users.noreply.github.com>
Date: Mon, 10 Oct 2022 16:44:43 +0200
Subject: [PATCH] Make the Store liveness probe timeout configurable

Under heavy load, the 1s default timeout for the Store liveness probe
can be often triggered, which leads to Store restarts.

The default value is 1s, to keep the current behavior, and in affected
environments one can increase it.

Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com>
---
 .../thanos-store-shard0-statefulSet.yaml          |  1 +
 .../thanos-store-shard1-statefulSet.yaml          |  1 +
 .../thanos-store-shard2-statefulSet.yaml          |  1 +
 .../all/manifests/thanos-store-statefulSet.yaml   |  1 +
 .../kube-thanos-store-default-params.libsonnet    |  3 +++
 jsonnet/kube-thanos/kube-thanos-store.libsonnet   | 15 ++++++++++-----
 manifests/thanos-store-statefulSet.yaml           |  1 +
 7 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/examples/all/manifests/thanos-store-shard0-statefulSet.yaml b/examples/all/manifests/thanos-store-shard0-statefulSet.yaml
index 07259f2..ed5080d 100644
--- a/examples/all/manifests/thanos-store-shard0-statefulSet.yaml
+++ b/examples/all/manifests/thanos-store-shard0-statefulSet.yaml
@@ -124,6 +124,7 @@ spec:
             port: 10902
             scheme: HTTP
           periodSeconds: 30
+          timeoutSeconds: 1
         name: thanos-store
         ports:
         - containerPort: 10901
diff --git a/examples/all/manifests/thanos-store-shard1-statefulSet.yaml b/examples/all/manifests/thanos-store-shard1-statefulSet.yaml
index 657a3c5..1c561ee 100644
--- a/examples/all/manifests/thanos-store-shard1-statefulSet.yaml
+++ b/examples/all/manifests/thanos-store-shard1-statefulSet.yaml
@@ -124,6 +124,7 @@ spec:
             port: 10902
             scheme: HTTP
           periodSeconds: 30
+          timeoutSeconds: 1
         name: thanos-store
         ports:
         - containerPort: 10901
diff --git a/examples/all/manifests/thanos-store-shard2-statefulSet.yaml b/examples/all/manifests/thanos-store-shard2-statefulSet.yaml
index 0a76de0..7d9a135 100644
--- a/examples/all/manifests/thanos-store-shard2-statefulSet.yaml
+++ b/examples/all/manifests/thanos-store-shard2-statefulSet.yaml
@@ -124,6 +124,7 @@ spec:
             port: 10902
             scheme: HTTP
           periodSeconds: 30
+          timeoutSeconds: 1
         name: thanos-store
         ports:
         - containerPort: 10901
diff --git a/examples/all/manifests/thanos-store-statefulSet.yaml b/examples/all/manifests/thanos-store-statefulSet.yaml
index 4e162fb..7ea32dc 100644
--- a/examples/all/manifests/thanos-store-statefulSet.yaml
+++ b/examples/all/manifests/thanos-store-statefulSet.yaml
@@ -112,6 +112,7 @@ spec:
             port: 10902
             scheme: HTTP
           periodSeconds: 30
+          timeoutSeconds: 1
         name: thanos-store
         ports:
         - containerPort: 10901
diff --git a/jsonnet/kube-thanos/kube-thanos-store-default-params.libsonnet b/jsonnet/kube-thanos/kube-thanos-store-default-params.libsonnet
index a989c0c..e288416 100644
--- a/jsonnet/kube-thanos/kube-thanos-store-default-params.libsonnet
+++ b/jsonnet/kube-thanos/kube-thanos-store-default-params.libsonnet
@@ -22,6 +22,9 @@
     grpc: 10901,
     http: 10902,
   },
+  livenessProbe: {
+    timeoutSeconds: 1,
+  },
   tracing: {},
   minTime: '',
   maxTime: '',
diff --git a/jsonnet/kube-thanos/kube-thanos-store.libsonnet b/jsonnet/kube-thanos/kube-thanos-store.libsonnet
index faad418..d226496 100644
--- a/jsonnet/kube-thanos/kube-thanos-store.libsonnet
+++ b/jsonnet/kube-thanos/kube-thanos-store.libsonnet
@@ -135,11 +135,16 @@ function(params) {
           { name: 'tls-secret', mountPath: ts.config.objectStorageConfig.tlsSecretMountPath },
         ] else []
       ),
-      livenessProbe: { failureThreshold: 8, periodSeconds: 30, httpGet: {
-        scheme: 'HTTP',
-        port: ts.config.ports.http,
-        path: '/-/healthy',
-      } },
+      livenessProbe: {
+        failureThreshold: 8,
+        periodSeconds: 30,
+        timeoutSeconds: ts.config.livenessProbe.timeoutSeconds,
+        httpGet: {
+          scheme: 'HTTP',
+          port: ts.config.ports.http,
+          path: '/-/healthy',
+        },
+      },
       readinessProbe: { failureThreshold: 20, periodSeconds: 5, httpGet: {
         scheme: 'HTTP',
         port: ts.config.ports.http,
diff --git a/manifests/thanos-store-statefulSet.yaml b/manifests/thanos-store-statefulSet.yaml
index 7da4ae2..34d0eb9 100644
--- a/manifests/thanos-store-statefulSet.yaml
+++ b/manifests/thanos-store-statefulSet.yaml
@@ -71,6 +71,7 @@ spec:
             port: 10902
             scheme: HTTP
           periodSeconds: 30
+          timeoutSeconds: 1
         name: thanos-store
         ports:
         - containerPort: 10901