From 6a439bdffa04c8aaa13af6cf998cb5073d11e9d2 Mon Sep 17 00:00:00 2001
From: zenador <zenador@users.noreply.github.com>
Date: Tue, 18 Mar 2025 15:42:28 +0800
Subject: [PATCH] remote read: Use more informative error msg for timeouts
 (#16157)

* Use more informative error msg for timeouts on remote read

Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>
---
 storage/remote/client.go      |  3 ++-
 storage/remote/client_test.go | 13 ++++++++++++-
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/storage/remote/client.go b/storage/remote/client.go
index b25d64529b..714ee872e9 100644
--- a/storage/remote/client.go
+++ b/storage/remote/client.go
@@ -365,7 +365,8 @@ func (c *Client) Read(ctx context.Context, query *prompb.Query, sortSeries bool)
 	httpReq.Header.Set("User-Agent", UserAgent)
 	httpReq.Header.Set("X-Prometheus-Remote-Read-Version", "0.1.0")
 
-	ctx, cancel := context.WithTimeout(ctx, c.timeout)
+	errTimeout := fmt.Errorf("%w: request timed out after %s", context.DeadlineExceeded, c.timeout)
+	ctx, cancel := context.WithTimeoutCause(ctx, c.timeout, errTimeout)
 
 	ctx, span := otel.Tracer("").Start(ctx, "Remote Read", trace.WithSpanKind(trace.SpanKindClient))
 	defer span.End()
diff --git a/storage/remote/client_test.go b/storage/remote/client_test.go
index 9fd8680cde..7753de04a9 100644
--- a/storage/remote/client_test.go
+++ b/storage/remote/client_test.go
@@ -220,6 +220,7 @@ func TestReadClient(t *testing.T) {
 		name                  string
 		query                 *prompb.Query
 		httpHandler           http.HandlerFunc
+		timeout               time.Duration
 		expectedLabels        []map[string]string
 		expectedSamples       [][]model.SamplePair
 		expectedErrorContains string
@@ -329,6 +330,12 @@ func TestReadClient(t *testing.T) {
 			}),
 			expectedErrorContains: "unsupported content type",
 		},
+		{
+			name:                  "timeout",
+			httpHandler:           sampledResponseHTTPHandler(t),
+			timeout:               time.Nanosecond,
+			expectedErrorContains: "context deadline exceeded: request timed out after 1ns",
+		},
 	}
 
 	for _, test := range tests {
@@ -339,9 +346,13 @@ func TestReadClient(t *testing.T) {
 			u, err := url.Parse(server.URL)
 			require.NoError(t, err)
 
+			if test.timeout == 0 {
+				test.timeout = 5 * time.Second
+			}
+
 			conf := &ClientConfig{
 				URL:              &config_util.URL{URL: u},
-				Timeout:          model.Duration(5 * time.Second),
+				Timeout:          model.Duration(test.timeout),
 				ChunkedReadLimit: config.DefaultChunkedReadLimit,
 			}
 			c, err := NewReadClient("test", conf)