scraping: Create a span and send the traceparent header during scrape requests (#16425)

* Traceparent header

Signed-off-by: Vanshikav123 <vanshikav928@gmail.com>
(cherry picked from commit 44a620dd73)
Signed-off-by: Vanshikav123 <vanshikav928@gmail.com>

* changes

Signed-off-by: Vanshikav123 <vanshikav928@gmail.com>
(cherry picked from commit 6e98a77b2d)
Signed-off-by: Vanshikav123 <vanshikav928@gmail.com>

* adding test

Signed-off-by: Vanshikav123 <vanshikav928@gmail.com>
(cherry picked from commit 97f288ad87)
Signed-off-by: Vanshikav123 <vanshikav928@gmail.com>

* more changes

Signed-off-by: Vanshikav123 <vanshikav928@gmail.com>
(cherry picked from commit d5dd861544)
Signed-off-by: Vanshikav123 <vanshikav928@gmail.com>

* extract http client creation to newScrapeClient

Signed-off-by: Vanshikav123 <vanshikav928@gmail.com>
(cherry picked from commit 3cd8092b155df069d02d9409b6327fe60c788bec)
Signed-off-by: Vanshikav123 <vanshikav928@gmail.com>

* rebase

Signed-off-by: Vanshikav123 <vanshikav928@gmail.com>

* rebase

Signed-off-by: Vanshikav123 <vanshikav928@gmail.com>

* reverting

Signed-off-by: Vanshikav123 <vanshikav928@gmail.com>

* ctx

Signed-off-by: Vanshikav123 <vanshikav928@gmail.com>

---------

Signed-off-by: Vanshikav123 <vanshikav928@gmail.com>
This commit is contained in:
vanshika 2025-07-15 13:07:24 +05:30 committed by GitHub
parent 3d245e31d3
commit 0fc5e75504
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 78 additions and 4 deletions

View File

@ -23,6 +23,7 @@ import (
"log/slog"
"math"
"net/http"
"net/http/httptrace"
"reflect"
"slices"
"strconv"
@ -36,6 +37,10 @@ import (
"github.com/prometheus/common/model"
"github.com/prometheus/common/promslog"
"github.com/prometheus/common/version"
"go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace"
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/trace"
"github.com/prometheus/prometheus/config"
"github.com/prometheus/prometheus/discovery/targetgroup"
@ -144,9 +149,9 @@ func newScrapePool(cfg *config.ScrapeConfig, app storage.Appendable, offsetSeed
logger = promslog.NewNopLogger()
}
client, err := config_util.NewClientFromConfig(cfg.HTTPClientConfig, cfg.JobName, options.HTTPClientOptions...)
client, err := newScrapeClient(cfg.HTTPClientConfig, cfg.JobName, options.HTTPClientOptions...)
if err != nil {
return nil, fmt.Errorf("error creating HTTP client: %w", err)
return nil, err
}
var escapingScheme model.EscapingScheme
@ -311,10 +316,10 @@ func (sp *scrapePool) reload(cfg *config.ScrapeConfig) error {
sp.metrics.targetScrapePoolReloads.Inc()
start := time.Now()
client, err := config_util.NewClientFromConfig(cfg.HTTPClientConfig, cfg.JobName, sp.httpOpts...)
client, err := newScrapeClient(cfg.HTTPClientConfig, cfg.JobName, sp.httpOpts...)
if err != nil {
sp.metrics.targetScrapePoolReloadsFailed.Inc()
return fmt.Errorf("error creating HTTP client: %w", err)
return err
}
reuseCache := reusableCache(sp.config, cfg)
@ -824,6 +829,8 @@ func (s *targetScraper) scrape(ctx context.Context) (*http.Response, error) {
s.req = req
}
ctx, span := otel.Tracer("").Start(ctx, "Scrape", trace.WithSpanKind(trace.SpanKindClient))
defer span.End()
return s.client.Do(s.req.WithContext(ctx))
}
@ -2268,3 +2275,16 @@ func pickSchema(bucketFactor float64) int32 {
return int32(floor)
}
}
func newScrapeClient(cfg config_util.HTTPClientConfig, name string, optFuncs ...config_util.HTTPClientOption) (*http.Client, error) {
client, err := config_util.NewClientFromConfig(cfg, name, optFuncs...)
if err != nil {
return nil, fmt.Errorf("error creating HTTP client: %w", err)
}
client.Transport = otelhttp.NewTransport(
client.Transport,
otelhttp.WithClientTrace(func(ctx context.Context) *httptrace.ClientTrace {
return otelhttptrace.NewClientTrace(ctx, otelhttptrace.WithoutSubSpans())
}))
return client, nil
}

View File

@ -45,6 +45,9 @@ import (
"github.com/prometheus/common/model"
"github.com/prometheus/common/promslog"
"github.com/stretchr/testify/require"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/propagation"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
"github.com/prometheus/prometheus/config"
"github.com/prometheus/prometheus/discovery"
@ -3112,6 +3115,57 @@ func TestAcceptHeader(t *testing.T) {
}
}
// setupTracing temporarily sets the global TracerProvider and Propagator
// and restores the original state after the test completes.
func setupTracing(t *testing.T) {
t.Helper()
origTracerProvider := otel.GetTracerProvider()
origPropagator := otel.GetTextMapPropagator()
tp := sdktrace.NewTracerProvider(sdktrace.WithSampler(sdktrace.AlwaysSample()))
otel.SetTracerProvider(tp)
otel.SetTextMapPropagator(propagation.TraceContext{})
t.Cleanup(func() {
otel.SetTracerProvider(origTracerProvider)
otel.SetTextMapPropagator(origPropagator)
})
}
// TestRequestTraceparentHeader verifies that the HTTP client used by the target scraper
// propagates the OpenTelemetry "traceparent" header correctly.
func TestRequestTraceparentHeader(t *testing.T) {
setupTracing(t)
server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
// the traceparent header is sent.
require.NotEmpty(t, r.Header.Get("traceparent"))
}))
defer server.Close()
serverURL, err := url.Parse(server.URL)
require.NoError(t, err)
client, err := newScrapeClient(config_util.DefaultHTTPClientConfig, "test")
require.NoError(t, err)
ts := &targetScraper{
Target: &Target{
labels: labels.FromStrings(
model.SchemeLabel, serverURL.Scheme,
model.AddressLabel, serverURL.Host,
),
scrapeConfig: &config.ScrapeConfig{},
},
client: client,
}
resp, err := ts.scrape(context.Background())
require.NoError(t, err)
require.NotNil(t, resp)
defer resp.Body.Close()
}
func TestTargetScraperScrapeOK(t *testing.T) {
const (
configTimeout = 1500 * time.Millisecond