From 0fc5e75504d876d041fef071097d8239edb29e44 Mon Sep 17 00:00:00 2001 From: vanshika <102902652+Vperiodt@users.noreply.github.com> Date: Tue, 15 Jul 2025 13:07:24 +0530 Subject: [PATCH] scraping: Create a span and send the traceparent header during scrape requests (#16425) * Traceparent header Signed-off-by: Vanshikav123 (cherry picked from commit 44a620dd733847a1711bac6bedc9731cda4aace0) Signed-off-by: Vanshikav123 * changes Signed-off-by: Vanshikav123 (cherry picked from commit 6e98a77b2d82a524462c39476a4e35ec6ce9e738) Signed-off-by: Vanshikav123 * adding test Signed-off-by: Vanshikav123 (cherry picked from commit 97f288ad87e82515794daed34fbc34a1d6a3bcf7) Signed-off-by: Vanshikav123 * more changes Signed-off-by: Vanshikav123 (cherry picked from commit d5dd861544f941cbd31189fb544e6125186ac2a1) Signed-off-by: Vanshikav123 * extract http client creation to newScrapeClient Signed-off-by: Vanshikav123 (cherry picked from commit 3cd8092b155df069d02d9409b6327fe60c788bec) Signed-off-by: Vanshikav123 * rebase Signed-off-by: Vanshikav123 * rebase Signed-off-by: Vanshikav123 * reverting Signed-off-by: Vanshikav123 * ctx Signed-off-by: Vanshikav123 --------- Signed-off-by: Vanshikav123 --- scrape/scrape.go | 28 ++++++++++++++++++---- scrape/scrape_test.go | 54 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 4 deletions(-) diff --git a/scrape/scrape.go b/scrape/scrape.go index 34f74b5495..b4f34a6f5b 100644 --- a/scrape/scrape.go +++ b/scrape/scrape.go @@ -23,6 +23,7 @@ import ( "log/slog" "math" "net/http" + "net/http/httptrace" "reflect" "slices" "strconv" @@ -36,6 +37,10 @@ import ( "github.com/prometheus/common/model" "github.com/prometheus/common/promslog" "github.com/prometheus/common/version" + "go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/trace" "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/discovery/targetgroup" @@ -144,9 +149,9 @@ func newScrapePool(cfg *config.ScrapeConfig, app storage.Appendable, offsetSeed logger = promslog.NewNopLogger() } - client, err := config_util.NewClientFromConfig(cfg.HTTPClientConfig, cfg.JobName, options.HTTPClientOptions...) + client, err := newScrapeClient(cfg.HTTPClientConfig, cfg.JobName, options.HTTPClientOptions...) if err != nil { - return nil, fmt.Errorf("error creating HTTP client: %w", err) + return nil, err } var escapingScheme model.EscapingScheme @@ -311,10 +316,10 @@ func (sp *scrapePool) reload(cfg *config.ScrapeConfig) error { sp.metrics.targetScrapePoolReloads.Inc() start := time.Now() - client, err := config_util.NewClientFromConfig(cfg.HTTPClientConfig, cfg.JobName, sp.httpOpts...) + client, err := newScrapeClient(cfg.HTTPClientConfig, cfg.JobName, sp.httpOpts...) if err != nil { sp.metrics.targetScrapePoolReloadsFailed.Inc() - return fmt.Errorf("error creating HTTP client: %w", err) + return err } reuseCache := reusableCache(sp.config, cfg) @@ -824,6 +829,8 @@ func (s *targetScraper) scrape(ctx context.Context) (*http.Response, error) { s.req = req } + ctx, span := otel.Tracer("").Start(ctx, "Scrape", trace.WithSpanKind(trace.SpanKindClient)) + defer span.End() return s.client.Do(s.req.WithContext(ctx)) } @@ -2268,3 +2275,16 @@ func pickSchema(bucketFactor float64) int32 { return int32(floor) } } + +func newScrapeClient(cfg config_util.HTTPClientConfig, name string, optFuncs ...config_util.HTTPClientOption) (*http.Client, error) { + client, err := config_util.NewClientFromConfig(cfg, name, optFuncs...) + if err != nil { + return nil, fmt.Errorf("error creating HTTP client: %w", err) + } + client.Transport = otelhttp.NewTransport( + client.Transport, + otelhttp.WithClientTrace(func(ctx context.Context) *httptrace.ClientTrace { + return otelhttptrace.NewClientTrace(ctx, otelhttptrace.WithoutSubSpans()) + })) + return client, nil +} diff --git a/scrape/scrape_test.go b/scrape/scrape_test.go index 2053e612c2..c226be2d33 100644 --- a/scrape/scrape_test.go +++ b/scrape/scrape_test.go @@ -45,6 +45,9 @@ import ( "github.com/prometheus/common/model" "github.com/prometheus/common/promslog" "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" + sdktrace "go.opentelemetry.io/otel/sdk/trace" "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/discovery" @@ -3112,6 +3115,57 @@ func TestAcceptHeader(t *testing.T) { } } +// setupTracing temporarily sets the global TracerProvider and Propagator +// and restores the original state after the test completes. +func setupTracing(t *testing.T) { + t.Helper() + + origTracerProvider := otel.GetTracerProvider() + origPropagator := otel.GetTextMapPropagator() + + tp := sdktrace.NewTracerProvider(sdktrace.WithSampler(sdktrace.AlwaysSample())) + otel.SetTracerProvider(tp) + otel.SetTextMapPropagator(propagation.TraceContext{}) + + t.Cleanup(func() { + otel.SetTracerProvider(origTracerProvider) + otel.SetTextMapPropagator(origPropagator) + }) +} + +// TestRequestTraceparentHeader verifies that the HTTP client used by the target scraper +// propagates the OpenTelemetry "traceparent" header correctly. +func TestRequestTraceparentHeader(t *testing.T) { + setupTracing(t) + + server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + // the traceparent header is sent. + require.NotEmpty(t, r.Header.Get("traceparent")) + })) + defer server.Close() + serverURL, err := url.Parse(server.URL) + require.NoError(t, err) + + client, err := newScrapeClient(config_util.DefaultHTTPClientConfig, "test") + require.NoError(t, err) + + ts := &targetScraper{ + Target: &Target{ + labels: labels.FromStrings( + model.SchemeLabel, serverURL.Scheme, + model.AddressLabel, serverURL.Host, + ), + scrapeConfig: &config.ScrapeConfig{}, + }, + client: client, + } + + resp, err := ts.scrape(context.Background()) + require.NoError(t, err) + require.NotNil(t, resp) + defer resp.Body.Close() +} + func TestTargetScraperScrapeOK(t *testing.T) { const ( configTimeout = 1500 * time.Millisecond