remote read: Use more informative error msg for timeouts (#16157)

* Use more informative error msg for timeouts on remote read

Signed-off-by: Jeanette Tan <jeanette.tan@grafana.com>

storage/remote/client.go

@@ -365,7 +365,8 @@ func (c *Client) Read(ctx context.Context, query *prompb.Query, sortSeries bool)
 	httpReq.Header.Set("User-Agent", UserAgent)
 	httpReq.Header.Set("X-Prometheus-Remote-Read-Version", "0.1.0")
 
-	ctx, cancel := context.WithTimeout(ctx, c.timeout)
+	errTimeout := fmt.Errorf("%w: request timed out after %s", context.DeadlineExceeded, c.timeout)
+	ctx, cancel := context.WithTimeoutCause(ctx, c.timeout, errTimeout)
 
 	ctx, span := otel.Tracer("").Start(ctx, "Remote Read", trace.WithSpanKind(trace.SpanKindClient))
 	defer span.End()

storage/remote/client_test.go

@@ -220,6 +220,7 @@ func TestReadClient(t *testing.T) {
 		name                  string
 		query                 *prompb.Query
 		httpHandler           http.HandlerFunc
+		timeout               time.Duration
 		expectedLabels        []map[string]string
 		expectedSamples       [][]model.SamplePair
 		expectedErrorContains string
@@ -329,6 +330,12 @@ func TestReadClient(t *testing.T) {
 			}),
 			expectedErrorContains: "unsupported content type",
 		},
+		{
+			name:                  "timeout",
+			httpHandler:           sampledResponseHTTPHandler(t),
+			timeout:               time.Nanosecond,
+			expectedErrorContains: "context deadline exceeded: request timed out after 1ns",
+		},
 	}
 
 	for _, test := range tests {
@@ -339,9 +346,13 @@ func TestReadClient(t *testing.T) {
 			u, err := url.Parse(server.URL)
 			require.NoError(t, err)
 
+			if test.timeout == 0 {
+				test.timeout = 5 * time.Second
+			}
+
 			conf := &ClientConfig{
 				URL:              &config_util.URL{URL: u},
-				Timeout:          model.Duration(5 * time.Second),
+				Timeout:          model.Duration(test.timeout),
 				ChunkedReadLimit: config.DefaultChunkedReadLimit,
 			}
 			c, err := NewReadClient("test", conf)