From 0daa2dbf59d9c9847ba3c23fef58f33e19c171cc Mon Sep 17 00:00:00 2001
From: Andreas Auernhammer <github@aead.dev>
Date: Thu, 16 Nov 2023 10:51:27 +0100
Subject: [PATCH] health: split liveness and readiness handler (#18457)

This commit splits the liveness and readiness
handler into two separate handlers. In K8S, a
liveness probe is used to determine whether the
pod is in "live" state and functioning at all.
In contrast, the readiness probe is used to
determine whether the pod is ready to serve
requests.

A failing liveness probe causes pod restarts while
a failing readiness probe causes k8s to stop routing
traffic to the pod. Hence, a liveness probe should
be as robust as possible while a readiness probe
should be used to load balancing.

Ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/

Signed-off-by: Andreas Auernhammer <github@aead.dev>
---
 cmd/healthcheck-handler.go | 48 ++++++++++++++++++++++++++------------
 1 file changed, 33 insertions(+), 15 deletions(-)

diff --git a/cmd/healthcheck-handler.go b/cmd/healthcheck-handler.go
index b9ed1d28a..1e8be5743 100644
--- a/cmd/healthcheck-handler.go
+++ b/cmd/healthcheck-handler.go
@@ -90,21 +90,15 @@ func ClusterReadCheckHandler(w http.ResponseWriter, r *http.Request) {
 	writeResponse(w, http.StatusOK, nil, mimeNone)
 }
 
-// ReadinessCheckHandler Checks if the process is up. Always returns success.
+// ReadinessCheckHandler checks whether MinIO is up and ready to serve requests.
+// It also checks whether the KMS is available and whether etcd is reachable,
+// if configured.
 func ReadinessCheckHandler(w http.ResponseWriter, r *http.Request) {
-	LivenessCheckHandler(w, r)
-}
-
-// LivenessCheckHandler - Checks if the process is up. Always returns success.
-func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) {
-	objLayer := newObjectLayerFn()
-	if objLayer == nil {
-		// Service not initialized yet
-		w.Header().Set(xhttp.MinIOServerStatus, unavailable)
+	if objLayer := newObjectLayerFn(); objLayer == nil {
+		w.Header().Set(xhttp.MinIOServerStatus, unavailable) // Service not initialized yet
 	}
-
-	peerCall := r.Header.Get(xhttp.MinIOPeerCall) != ""
-	if peerCall {
+	if r.Header.Get(xhttp.MinIOPeerCall) != "" {
+		writeResponse(w, http.StatusOK, nil, mimeNone)
 		return
 	}
 
@@ -120,7 +114,7 @@ func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// Verify if KMS is reachable if its configured
-	if GlobalKMS != nil && !peerCall {
+	if GlobalKMS != nil {
 		ctx, cancel := context.WithTimeout(r.Context(), time.Minute)
 		defer cancel()
 
@@ -153,6 +147,30 @@ func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 	}
-
+	writeResponse(w, http.StatusOK, nil, mimeNone)
+}
+
+// LivenessCheckHandler checks whether MinIO is up. It differs from the
+// readiness handler since a failing liveness check causes pod restarts
+// in K8S enviromnents. Therefore, it does not contact external systems.
+func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) {
+	if objLayer := newObjectLayerFn(); objLayer == nil {
+		w.Header().Set(xhttp.MinIOServerStatus, unavailable) // Service not initialized yet
+	}
+	if r.Header.Get(xhttp.MinIOPeerCall) != "" {
+		writeResponse(w, http.StatusOK, nil, mimeNone)
+		return
+	}
+
+	if int(globalHTTPStats.loadRequestsInQueue()) > globalAPIConfig.getRequestsPoolCapacity() {
+		apiErr := getAPIError(ErrBusy)
+		switch r.Method {
+		case http.MethodHead:
+			writeResponse(w, apiErr.HTTPStatusCode, nil, mimeNone)
+		case http.MethodGet:
+			writeErrorResponse(r.Context(), w, apiErr, r.URL)
+		}
+		return
+	}
 	writeResponse(w, http.StatusOK, nil, mimeNone)
 }