mirrors/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2025-08-05 21:57:09 +02:00
Code Issues Projects Releases Wiki Activity

Support including scope metadata as metric labels (#16878)

Browse Source 
* Support including scope metadata as metric labels

Signed-off-by: Arthur Silva Sens <arthursens2005@gmail.com>

* Ensure Scope Name, Version and Schema URL aren't overriden by attributes

Signed-off-by: Arthur Silva Sens <arthursens2005@gmail.com>

---------

Signed-off-by: Arthur Silva Sens <arthursens2005@gmail.com>
This commit is contained in:
Arthur Silva Sens 2025-07-18 13:44:05 -03:00 committed by GitHub
parent 46acc974c0
commit 3602785a89
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 831 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
config/config.go
View File

@ -1604,6 +1604,9 @@ type OTLPConfig struct {
TranslationStrategy translationStrategyOption `yaml:"translation_strategy,omitempty"` TranslationStrategy translationStrategyOption `yaml:"translation_strategy,omitempty"`
KeepIdentifyingResourceAttributes bool `yaml:"keep_identifying_resource_attributes,omitempty"` KeepIdentifyingResourceAttributes bool `yaml:"keep_identifying_resource_attributes,omitempty"`
ConvertHistogramsToNHCB bool `yaml:"convert_histograms_to_nhcb,omitempty"` ConvertHistogramsToNHCB bool `yaml:"convert_histograms_to_nhcb,omitempty"`
// PromoteScopeMetadata controls whether to promote OTel scope metadata (i.e. name, version, schema URL, and attributes) to metric labels.
// As per OTel spec, the aforementioned scope metadata should be identifying, i.e. made into metric labels.
PromoteScopeMetadata bool `yaml:"promote_scope_metadata,omitempty"`
} }
// UnmarshalYAML implements the yaml.Unmarshaler interface. // UnmarshalYAML implements the yaml.Unmarshaler interface.

14
config/config_test.go
View File

@ -1808,6 +1808,20 @@ func TestOTLPConvertHistogramsToNHCB(t *testing.T) {
}) })
} }
func TestOTLPPromoteScopeMetadata(t *testing.T) {
t.Run("good config", func(t *testing.T) {
want, err := LoadFile(filepath.Join("testdata", "otlp_promote_scope_metadata.good.yml"), false, promslog.NewNopLogger())
require.NoError(t, err)
out, err := yaml.Marshal(want)
require.NoError(t, err)
var got Config
require.NoError(t, yaml.UnmarshalStrict(out, &got))
require.True(t, got.OTLPConfig.PromoteScopeMetadata)
})
}
func TestOTLPAllowUTF8(t *testing.T) { func TestOTLPAllowUTF8(t *testing.T) {
t.Run("good config - NoUTF8EscapingWithSuffixes", func(t *testing.T) { t.Run("good config - NoUTF8EscapingWithSuffixes", func(t *testing.T) {
fpath := filepath.Join("testdata", "otlp_allow_utf8.good.yml") fpath := filepath.Join("testdata", "otlp_allow_utf8.good.yml")

2
config/testdata/otlp_promote_scope_metadata.good.yml vendored Normal file
View File

@ -0,0 +1,2 @@
otlp:
promote_scope_metadata: true

3
docs/configuration/configuration.md
View File

@ -219,6 +219,9 @@ otlp:
[ keep_identifying_resource_attributes: <boolean> | default = false ] [ keep_identifying_resource_attributes: <boolean> | default = false ]
# Configures optional translation of OTLP explicit bucket histograms into native histograms with custom buckets. # Configures optional translation of OTLP explicit bucket histograms into native histograms with custom buckets.
[ convert_histograms_to_nhcb: <boolean> | default = false ] [ convert_histograms_to_nhcb: <boolean> | default = false ]
# Enables promotion of OTel scope metadata (i.e. name, version, schema URL, and attributes) to metric labels.
# This is disabled by default for backwards compatibility, but according to OTel spec, scope metadata _should_ be identifying, i.e. translated to metric labels.
[ promote_scope_metadata: <boolean> | default = false ]
# Settings related to the remote read feature. # Settings related to the remote read feature.
remote_read: remote_read:

32
storage/remote/otlptranslator/prometheusremotewrite/helper.go
View File

@ -117,7 +117,7 @@ var seps = []byte{'\xff'}
// Unpaired string values are ignored. String pairs overwrite OTLP labels if collisions happen and // Unpaired string values are ignored. String pairs overwrite OTLP labels if collisions happen and
// if logOnOverwrite is true, the overwrite is logged. Resulting label names are sanitized. // if logOnOverwrite is true, the overwrite is logged. Resulting label names are sanitized.
// If settings.PromoteResourceAttributes is not empty, it's a set of resource attributes that should be promoted to labels. // If settings.PromoteResourceAttributes is not empty, it's a set of resource attributes that should be promoted to labels.
func createAttributes(resource pcommon.Resource, attributes pcommon.Map, settings Settings, func createAttributes(resource pcommon.Resource, attributes pcommon.Map, scope scope, settings Settings,
ignoreAttrs []string, logOnOverwrite bool, extras ...string, ignoreAttrs []string, logOnOverwrite bool, extras ...string,
) []prompb.Label { ) []prompb.Label {
resourceAttrs := resource.Attributes() resourceAttrs := resource.Attributes()
@ -126,8 +126,15 @@ func createAttributes(resource pcommon.Resource, attributes pcommon.Map, setting
promotedAttrs := settings.PromoteResourceAttributes.promotedAttributes(resourceAttrs) promotedAttrs := settings.PromoteResourceAttributes.promotedAttributes(resourceAttrs)
promoteScope := settings.PromoteScopeMetadata && scope.name != ""
scopeLabelCount := 0
if promoteScope {
// Include name, version and schema URL.
scopeLabelCount = scope.attributes.Len() + 3
}
// Calculate the maximum possible number of labels we could return so we can preallocate l. // Calculate the maximum possible number of labels we could return so we can preallocate l.
maxLabelCount := attributes.Len() + len(settings.ExternalLabels) + len(promotedAttrs) + len(extras)/2 maxLabelCount := attributes.Len() + len(settings.ExternalLabels) + len(promotedAttrs) + scopeLabelCount + len(extras)/2
if haveServiceName { if haveServiceName {
maxLabelCount++ maxLabelCount++
@ -167,6 +174,17 @@ func createAttributes(resource pcommon.Resource, attributes pcommon.Map, setting
l[normalized] = lbl.Value l[normalized] = lbl.Value
} }
} }
if promoteScope {
scope.attributes.Range(func(k string, v pcommon.Value) bool {
name := labelNamer.Build("otel_scope_" + k)
l[name] = v.AsString()
return true
})
// Scope Name, Version and Schema URL are added after attributes to ensure they are not overwritten by attributes.
l["otel_scope_name"] = scope.name
l["otel_scope_version"] = scope.version
l["otel_scope_schema_url"] = scope.schemaURL
}
// Map service.name + service.namespace to job. // Map service.name + service.namespace to job.
if haveServiceName { if haveServiceName {
@ -237,7 +255,7 @@ func aggregationTemporality(metric pmetric.Metric) (pmetric.AggregationTemporali
// However, work is under way to resolve this shortcoming through a feature called native histograms custom buckets: // However, work is under way to resolve this shortcoming through a feature called native histograms custom buckets:
// https://github.com/prometheus/prometheus/issues/13485. // https://github.com/prometheus/prometheus/issues/13485.
func (c *PrometheusConverter) addHistogramDataPoints(ctx context.Context, dataPoints pmetric.HistogramDataPointSlice, func (c *PrometheusConverter) addHistogramDataPoints(ctx context.Context, dataPoints pmetric.HistogramDataPointSlice,
resource pcommon.Resource, settings Settings, baseName string, resource pcommon.Resource, settings Settings, baseName string, scope scope,
) error { ) error {
for x := 0; x < dataPoints.Len(); x++ { for x := 0; x < dataPoints.Len(); x++ {
if err := c.everyN.checkContext(ctx); err != nil { if err := c.everyN.checkContext(ctx); err != nil {
@ -246,7 +264,7 @@ func (c *PrometheusConverter) addHistogramDataPoints(ctx context.Context, dataPo
pt := dataPoints.At(x) pt := dataPoints.At(x)
timestamp := convertTimeStamp(pt.Timestamp()) timestamp := convertTimeStamp(pt.Timestamp())
baseLabels := createAttributes(resource, pt.Attributes(), settings, nil, false) baseLabels := createAttributes(resource, pt.Attributes(), scope, settings, nil, false)
// If the sum is unset, it indicates the _sum metric point should be // If the sum is unset, it indicates the _sum metric point should be
// omitted // omitted
@ -447,7 +465,7 @@ func findMinAndMaxTimestamps(metric pmetric.Metric, minTimestamp, maxTimestamp p
} }
func (c *PrometheusConverter) addSummaryDataPoints(ctx context.Context, dataPoints pmetric.SummaryDataPointSlice, resource pcommon.Resource, func (c *PrometheusConverter) addSummaryDataPoints(ctx context.Context, dataPoints pmetric.SummaryDataPointSlice, resource pcommon.Resource,
settings Settings, baseName string, settings Settings, baseName string, scope scope,
) error { ) error {
for x := 0; x < dataPoints.Len(); x++ { for x := 0; x < dataPoints.Len(); x++ {
if err := c.everyN.checkContext(ctx); err != nil { if err := c.everyN.checkContext(ctx); err != nil {
@ -456,7 +474,7 @@ func (c *PrometheusConverter) addSummaryDataPoints(ctx context.Context, dataPoin
pt := dataPoints.At(x) pt := dataPoints.At(x)
timestamp := convertTimeStamp(pt.Timestamp()) timestamp := convertTimeStamp(pt.Timestamp())
baseLabels := createAttributes(resource, pt.Attributes(), settings, nil, false) baseLabels := createAttributes(resource, pt.Attributes(), scope, settings, nil, false)
// treat sum as a sample in an individual TimeSeries // treat sum as a sample in an individual TimeSeries
sum := &prompb.Sample{ sum := &prompb.Sample{
@ -609,7 +627,7 @@ func addResourceTargetInfo(resource pcommon.Resource, settings Settings, earlies
// Do not pass identifying attributes as ignoreAttrs below. // Do not pass identifying attributes as ignoreAttrs below.
identifyingAttrs = nil identifyingAttrs = nil
} }
labels := createAttributes(resource, attributes, settings, identifyingAttrs, false, model.MetricNameLabel, name) labels := createAttributes(resource, attributes, scope{}, settings, identifyingAttrs, false, model.MetricNameLabel, name)
haveIdentifier := false haveIdentifier := false
for _, l := range labels { for _, l := range labels {
if l.Name == model.JobLabel || l.Name == model.InstanceLabel { if l.Name == model.JobLabel || l.Name == model.InstanceLabel {

461
storage/remote/otlptranslator/prometheusremotewrite/helper_test.go
View File

@ -43,6 +43,17 @@ func TestCreateAttributes(t *testing.T) {
// This one is for testing conflict with auto-generated instance attribute. // This one is for testing conflict with auto-generated instance attribute.
"instance": "resource value", "instance": "resource value",
} }
scopeAttrs := pcommon.NewMap()
scopeAttrs.FromRaw(map[string]any{
"attr1": "value1",
"attr2": "value2",
})
defaultScope := scope{
name: "test-scope",
version: "1.0.0",
schemaURL: "https://schema.com",
attributes: scopeAttrs,
}
resource := pcommon.NewResource() resource := pcommon.NewResource()
for k, v := range resourceAttrs { for k, v := range resourceAttrs {
@ -54,14 +65,18 @@ func TestCreateAttributes(t *testing.T) {
testCases := []struct { testCases := []struct {
name string name string
scope scope
promoteAllResourceAttributes bool promoteAllResourceAttributes bool
promoteResourceAttributes []string promoteResourceAttributes []string
promoteScope bool
ignoreResourceAttributes []string ignoreResourceAttributes []string
ignoreAttrs []string ignoreAttrs []string
expectedLabels []prompb.Label expectedLabels []prompb.Label
}{ }{
{ {
name: "Successful conversion without resource attribute promotion", name: "Successful conversion without resource attribute promotion and without scope promotion",
scope: defaultScope,
promoteScope: false,
promoteResourceAttributes: nil, promoteResourceAttributes: nil,
expectedLabels: []prompb.Label{ expectedLabels: []prompb.Label{
{ {
@ -86,6 +101,82 @@ func TestCreateAttributes(t *testing.T) {
}, },
}, },
}, },
{
name: "Successful conversion without resource attribute promotion and with scope promotion",
scope: defaultScope,
promoteScope: true,
promoteResourceAttributes: nil,
expectedLabels: []prompb.Label{
{
Name: "__name__",
Value: "test_metric",
},
{
Name: "instance",
Value: "service ID",
},
{
Name: "job",
Value: "service name",
},
{
Name: "metric_attr",
Value: "metric value",
},
{
Name: "metric_attr_other",
Value: "metric value other",
},
{
Name: "otel_scope_name",
Value: defaultScope.name,
},
{
Name: "otel_scope_schema_url",
Value: defaultScope.schemaURL,
},
{
Name: "otel_scope_version",
Value: defaultScope.version,
},
{
Name: "otel_scope_attr1",
Value: "value1",
},
{
Name: "otel_scope_attr2",
Value: "value2",
},
},
},
{
name: "Successful conversion without resource attribute promotion and with scope promotion, but without scope",
scope: scope{},
promoteResourceAttributes: nil,
promoteScope: true,
expectedLabels: []prompb.Label{
{
Name: "__name__",
Value: "test_metric",
},
{
Name: "instance",
Value: "service ID",
},
{
Name: "job",
Value: "service name",
},
{
Name: "metric_attr",
Value: "metric value",
},
{
Name: "metric_attr_other",
Value: "metric value other",
},
},
},
{ {
name: "Successful conversion with some attributes ignored", name: "Successful conversion with some attributes ignored",
promoteResourceAttributes: nil, promoteResourceAttributes: nil,
@ -110,8 +201,55 @@ func TestCreateAttributes(t *testing.T) {
}, },
}, },
{ {
name: "Successful conversion with resource attribute promotion", name: "Successful conversion with some attributes ignored and with scope promotion",
scope: defaultScope,
promoteScope: true,
promoteResourceAttributes: nil,
ignoreAttrs: []string{"metric-attr-other"},
expectedLabels: []prompb.Label{
{
Name: "__name__",
Value: "test_metric",
},
{
Name: "instance",
Value: "service ID",
},
{
Name: "job",
Value: "service name",
},
{
Name: "metric_attr",
Value: "metric value",
},
{
Name: "otel_scope_name",
Value: defaultScope.name,
},
{
Name: "otel_scope_version",
Value: defaultScope.version,
},
{
Name: "otel_scope_schema_url",
Value: defaultScope.schemaURL,
},
{
Name: "otel_scope_attr1",
Value: "value1",
},
{
Name: "otel_scope_attr2",
Value: "value2",
},
},
},
{
name: "Successful conversion with resource attribute promotion and with scope promotion",
scope: defaultScope,
promoteResourceAttributes: []string{"non-existent-attr", "existent-attr"}, promoteResourceAttributes: []string{"non-existent-attr", "existent-attr"},
promoteScope: true,
expectedLabels: []prompb.Label{ expectedLabels: []prompb.Label{
{ {
Name: "__name__", Name: "__name__",
@ -137,10 +275,32 @@ func TestCreateAttributes(t *testing.T) {
Name: "existent_attr", Name: "existent_attr",
Value: "resource value", Value: "resource value",
}, },
{
Name: "otel_scope_name",
Value: defaultScope.name,
},
{
Name: "otel_scope_version",
Value: defaultScope.version,
},
{
Name: "otel_scope_schema_url",
Value: defaultScope.schemaURL,
},
{
Name: "otel_scope_attr1",
Value: "value1",
},
{
Name: "otel_scope_attr2",
Value: "value2",
},
}, },
}, },
{ {
name: "Successful conversion with resource attribute promotion, conflicting resource attributes are ignored", name: "Successful conversion with resource attribute promotion and with scope promotion, conflicting resource attributes are ignored",
scope: defaultScope,
promoteScope: true,
promoteResourceAttributes: []string{"non-existent-attr", "existent-attr", "metric-attr", "job", "instance"}, promoteResourceAttributes: []string{"non-existent-attr", "existent-attr", "metric-attr", "job", "instance"},
expectedLabels: []prompb.Label{ expectedLabels: []prompb.Label{
{ {
@ -167,10 +327,32 @@ func TestCreateAttributes(t *testing.T) {
Name: "metric_attr_other", Name: "metric_attr_other",
Value: "metric value other", Value: "metric value other",
}, },
{
Name: "otel_scope_name",
Value: defaultScope.name,
},
{
Name: "otel_scope_version",
Value: defaultScope.version,
},
{
Name: "otel_scope_schema_url",
Value: defaultScope.schemaURL,
},
{
Name: "otel_scope_attr1",
Value: "value1",
},
{
Name: "otel_scope_attr2",
Value: "value2",
},
}, },
}, },
{ {
name: "Successful conversion with resource attribute promotion, attributes are only promoted once", name: "Successful conversion with resource attribute promotion and with scope promotion, attributes are only promoted once",
scope: defaultScope,
promoteScope: true,
promoteResourceAttributes: []string{"existent-attr", "existent-attr"}, promoteResourceAttributes: []string{"existent-attr", "existent-attr"},
expectedLabels: []prompb.Label{ expectedLabels: []prompb.Label{
{ {
@ -197,11 +379,33 @@ func TestCreateAttributes(t *testing.T) {
Name: "metric_attr_other", Name: "metric_attr_other",
Value: "metric value other", Value: "metric value other",
}, },
{
Name: "otel_scope_name",
Value: defaultScope.name,
},
{
Name: "otel_scope_version",
Value: defaultScope.version,
},
{
Name: "otel_scope_schema_url",
Value: defaultScope.schemaURL,
},
{
Name: "otel_scope_attr1",
Value: "value1",
},
{
Name: "otel_scope_attr2",
Value: "value2",
},
}, },
}, },
{ {
name: "Successful conversion promoting all resource attributes", name: "Successful conversion promoting all resource attributes and with scope promotion",
scope: defaultScope,
promoteAllResourceAttributes: true, promoteAllResourceAttributes: true,
promoteScope: true,
expectedLabels: []prompb.Label{ expectedLabels: []prompb.Label{
{ {
Name: "__name__", Name: "__name__",
@ -235,10 +439,32 @@ func TestCreateAttributes(t *testing.T) {
Name: "service_instance_id", Name: "service_instance_id",
Value: "service ID", Value: "service ID",
}, },
{
Name: "otel_scope_name",
Value: defaultScope.name,
},
{
Name: "otel_scope_version",
Value: defaultScope.version,
},
{
Name: "otel_scope_schema_url",
Value: defaultScope.schemaURL,
},
{
Name: "otel_scope_attr1",
Value: "value1",
},
{
Name: "otel_scope_attr2",
Value: "value2",
},
}, },
}, },
{ {
name: "Successful conversion promoting all resource attributes, ignoring 'service.instance.id'", name: "Successful conversion promoting all resource attributes and with scope promotion, ignoring 'service.instance.id'",
scope: defaultScope,
promoteScope: true,
promoteAllResourceAttributes: true, promoteAllResourceAttributes: true,
ignoreResourceAttributes: []string{ ignoreResourceAttributes: []string{
"service.instance.id", "service.instance.id",
@ -272,6 +498,26 @@ func TestCreateAttributes(t *testing.T) {
Name: "service_name", Name: "service_name",
Value: "service name", Value: "service name",
}, },
{
Name: "otel_scope_name",
Value: defaultScope.name,
},
{
Name: "otel_scope_version",
Value: defaultScope.version,
},
{
Name: "otel_scope_schema_url",
Value: defaultScope.schemaURL,
},
{
Name: "otel_scope_attr1",
Value: "value1",
},
{
Name: "otel_scope_attr2",
Value: "value2",
},
}, },
}, },
} }
@ -283,8 +529,9 @@ func TestCreateAttributes(t *testing.T) {
PromoteResourceAttributes: tc.promoteResourceAttributes, PromoteResourceAttributes: tc.promoteResourceAttributes,
IgnoreResourceAttributes: tc.ignoreResourceAttributes, IgnoreResourceAttributes: tc.ignoreResourceAttributes,
}), }),
PromoteScopeMetadata: tc.promoteScope,
} }
lbls := createAttributes(resource, attrs, settings, tc.ignoreAttrs, false, model.MetricNameLabel, "test_metric") lbls := createAttributes(resource, attrs, tc.scope, settings, tc.ignoreAttrs, false, model.MetricNameLabel, "test_metric")
require.ElementsMatch(t, lbls, tc.expectedLabels) require.ElementsMatch(t, lbls, tc.expectedLabels)
}) })
@ -310,14 +557,27 @@ func Test_convertTimeStamp(t *testing.T) {
} }
func TestPrometheusConverter_AddSummaryDataPoints(t *testing.T) { func TestPrometheusConverter_AddSummaryDataPoints(t *testing.T) {
scopeAttrs := pcommon.NewMap()
scopeAttrs.FromRaw(map[string]any{
"attr1": "value1",
"attr2": "value2",
})
defaultScope := scope{
name: "test-scope",
version: "1.0.0",
schemaURL: "https://schema.com",
attributes: scopeAttrs,
}
ts := pcommon.Timestamp(time.Now().UnixNano()) ts := pcommon.Timestamp(time.Now().UnixNano())
tests := []struct { tests := []struct {
name string name string
metric func() pmetric.Metric metric func() pmetric.Metric
want func() map[uint64]*prompb.TimeSeries scope scope
promoteScope bool
want func() map[uint64]*prompb.TimeSeries
}{ }{
{ {
name: "summary with start time", name: "summary with start time and without scope promotion",
metric: func() pmetric.Metric { metric: func() pmetric.Metric {
metric := pmetric.NewMetric() metric := pmetric.NewMetric()
metric.SetName("test_summary") metric.SetName("test_summary")
@ -329,6 +589,8 @@ func TestPrometheusConverter_AddSummaryDataPoints(t *testing.T) {
return metric return metric
}, },
scope: defaultScope,
promoteScope: false,
want: func() map[uint64]*prompb.TimeSeries { want: func() map[uint64]*prompb.TimeSeries {
countLabels := []prompb.Label{ countLabels := []prompb.Label{
{Name: model.MetricNameLabel, Value: "test_summary" + countStr}, {Name: model.MetricNameLabel, Value: "test_summary" + countStr},
@ -362,7 +624,79 @@ func TestPrometheusConverter_AddSummaryDataPoints(t *testing.T) {
}, },
}, },
{ {
name: "summary without start time", name: "summary with start time and with scope promotion",
metric: func() pmetric.Metric {
metric := pmetric.NewMetric()
metric.SetName("test_summary")
metric.SetEmptySummary()
dp := metric.Summary().DataPoints().AppendEmpty()
dp.SetTimestamp(ts)
dp.SetStartTimestamp(ts)
return metric
},
scope: defaultScope,
promoteScope: true,
want: func() map[uint64]*prompb.TimeSeries {
scopeLabels := []prompb.Label{
{
Name: "otel_scope_attr1",
Value: "value1",
},
{
Name: "otel_scope_attr2",
Value: "value2",
},
{
Name: "otel_scope_name",
Value: defaultScope.name,
},
{
Name: "otel_scope_schema_url",
Value: defaultScope.schemaURL,
},
{
Name: "otel_scope_version",
Value: defaultScope.version,
},
}
countLabels := append([]prompb.Label{
{Name: model.MetricNameLabel, Value: "test_summary" + countStr},
}, scopeLabels...)
sumLabels := append([]prompb.Label{
{Name: model.MetricNameLabel, Value: "test_summary" + sumStr},
}, scopeLabels...)
createdLabels := append([]prompb.Label{
{
Name: model.MetricNameLabel,
Value: "test_summary" + createdSuffix,
},
}, scopeLabels...)
return map[uint64]*prompb.TimeSeries{
timeSeriesSignature(countLabels): {
Labels: countLabels,
Samples: []prompb.Sample{
{Value: 0, Timestamp: convertTimeStamp(ts)},
},
},
timeSeriesSignature(sumLabels): {
Labels: sumLabels,
Samples: []prompb.Sample{
{Value: 0, Timestamp: convertTimeStamp(ts)},
},
},
timeSeriesSignature(createdLabels): {
Labels: createdLabels,
Samples: []prompb.Sample{
{Value: float64(convertTimeStamp(ts)), Timestamp: convertTimeStamp(ts)},
},
},
}
},
},
{
name: "summary without start time and without scope promotion",
metric: func() pmetric.Metric { metric: func() pmetric.Metric {
metric := pmetric.NewMetric() metric := pmetric.NewMetric()
metric.SetName("test_summary") metric.SetName("test_summary")
@ -373,6 +707,8 @@ func TestPrometheusConverter_AddSummaryDataPoints(t *testing.T) {
return metric return metric
}, },
scope: defaultScope,
promoteScope: false,
want: func() map[uint64]*prompb.TimeSeries { want: func() map[uint64]*prompb.TimeSeries {
countLabels := []prompb.Label{ countLabels := []prompb.Label{
{Name: model.MetricNameLabel, Value: "test_summary" + countStr}, {Name: model.MetricNameLabel, Value: "test_summary" + countStr},
@ -407,9 +743,11 @@ func TestPrometheusConverter_AddSummaryDataPoints(t *testing.T) {
metric.Summary().DataPoints(), metric.Summary().DataPoints(),
pcommon.NewResource(), pcommon.NewResource(),
Settings{ Settings{
ExportCreatedMetric: true, ExportCreatedMetric: true,
PromoteScopeMetadata: tt.promoteScope,
}, },
metric.Name(), metric.Name(),
tt.scope,
) )
testutil.RequireEqual(t, tt.want(), converter.unique) testutil.RequireEqual(t, tt.want(), converter.unique)
@ -419,14 +757,27 @@ func TestPrometheusConverter_AddSummaryDataPoints(t *testing.T) {
} }
func TestPrometheusConverter_AddHistogramDataPoints(t *testing.T) { func TestPrometheusConverter_AddHistogramDataPoints(t *testing.T) {
scopeAttrs := pcommon.NewMap()
scopeAttrs.FromRaw(map[string]any{
"attr1": "value1",
"attr2": "value2",
})
defaultScope := scope{
name: "test-scope",
version: "1.0.0",
schemaURL: "https://schema.com",
attributes: scopeAttrs,
}
ts := pcommon.Timestamp(time.Now().UnixNano()) ts := pcommon.Timestamp(time.Now().UnixNano())
tests := []struct { tests := []struct {
name string name string
metric func() pmetric.Metric metric func() pmetric.Metric
want func() map[uint64]*prompb.TimeSeries scope scope
promoteScope bool
want func() map[uint64]*prompb.TimeSeries
}{ }{
{ {
name: "histogram with start time", name: "histogram with start time and without scope promotion",
metric: func() pmetric.Metric { metric: func() pmetric.Metric {
metric := pmetric.NewMetric() metric := pmetric.NewMetric()
metric.SetName("test_hist") metric.SetName("test_hist")
@ -438,6 +789,8 @@ func TestPrometheusConverter_AddHistogramDataPoints(t *testing.T) {
return metric return metric
}, },
scope: defaultScope,
promoteScope: false,
want: func() map[uint64]*prompb.TimeSeries { want: func() map[uint64]*prompb.TimeSeries {
countLabels := []prompb.Label{ countLabels := []prompb.Label{
{Name: model.MetricNameLabel, Value: "test_hist" + countStr}, {Name: model.MetricNameLabel, Value: "test_hist" + countStr},
@ -471,6 +824,76 @@ func TestPrometheusConverter_AddHistogramDataPoints(t *testing.T) {
} }
}, },
}, },
{
name: "histogram with start time and with scope promotion",
metric: func() pmetric.Metric {
metric := pmetric.NewMetric()
metric.SetName("test_hist")
metric.SetEmptyHistogram().SetAggregationTemporality(pmetric.AggregationTemporalityCumulative)
pt := metric.Histogram().DataPoints().AppendEmpty()
pt.SetTimestamp(ts)
pt.SetStartTimestamp(ts)
return metric
},
scope: defaultScope,
promoteScope: true,
want: func() map[uint64]*prompb.TimeSeries {
scopeLabels := []prompb.Label{
{
Name: "otel_scope_attr1",
Value: "value1",
},
{
Name: "otel_scope_attr2",
Value: "value2",
},
{
Name: "otel_scope_name",
Value: defaultScope.name,
},
{
Name: "otel_scope_schema_url",
Value: defaultScope.schemaURL,
},
{
Name: "otel_scope_version",
Value: defaultScope.version,
},
}
countLabels := append([]prompb.Label{
{Name: model.MetricNameLabel, Value: "test_hist" + countStr},
}, scopeLabels...)
infLabels := append([]prompb.Label{
{Name: model.MetricNameLabel, Value: "test_hist_bucket"},
{Name: model.BucketLabel, Value: "+Inf"},
}, scopeLabels...)
createdLabels := append([]prompb.Label{
{Name: model.MetricNameLabel, Value: "test_hist" + createdSuffix},
}, scopeLabels...)
return map[uint64]*prompb.TimeSeries{
timeSeriesSignature(countLabels): {
Labels: countLabels,
Samples: []prompb.Sample{
{Value: 0, Timestamp: convertTimeStamp(ts)},
},
},
timeSeriesSignature(infLabels): {
Labels: infLabels,
Samples: []prompb.Sample{
{Value: 0, Timestamp: convertTimeStamp(ts)},
},
},
timeSeriesSignature(createdLabels): {
Labels: createdLabels,
Samples: []prompb.Sample{
{Value: float64(convertTimeStamp(ts)), Timestamp: convertTimeStamp(ts)},
},
},
}
},
},
{ {
name: "histogram without start time", name: "histogram without start time",
metric: func() pmetric.Metric { metric: func() pmetric.Metric {
@ -518,9 +941,11 @@ func TestPrometheusConverter_AddHistogramDataPoints(t *testing.T) {
metric.Histogram().DataPoints(), metric.Histogram().DataPoints(),
pcommon.NewResource(), pcommon.NewResource(),
Settings{ Settings{
ExportCreatedMetric: true, ExportCreatedMetric: true,
PromoteScopeMetadata: tt.promoteScope,
}, },
metric.Name(), metric.Name(),
tt.scope,
) )
require.Equal(t, tt.want(), converter.unique) require.Equal(t, tt.want(), converter.unique)

4
storage/remote/otlptranslator/prometheusremotewrite/histograms.go
View File

@ -37,6 +37,7 @@ const defaultZeroThreshold = 1e-128
// as native histogram samples. // as native histogram samples.
func (c *PrometheusConverter) addExponentialHistogramDataPoints(ctx context.Context, dataPoints pmetric.ExponentialHistogramDataPointSlice, func (c *PrometheusConverter) addExponentialHistogramDataPoints(ctx context.Context, dataPoints pmetric.ExponentialHistogramDataPointSlice,
resource pcommon.Resource, settings Settings, promName string, temporality pmetric.AggregationTemporality, resource pcommon.Resource, settings Settings, promName string, temporality pmetric.AggregationTemporality,
scope scope,
) (annotations.Annotations, error) { ) (annotations.Annotations, error) {
var annots annotations.Annotations var annots annotations.Annotations
for x := 0; x < dataPoints.Len(); x++ { for x := 0; x < dataPoints.Len(); x++ {
@ -55,6 +56,7 @@ func (c *PrometheusConverter) addExponentialHistogramDataPoints(ctx context.Cont
lbls := createAttributes( lbls := createAttributes(
resource, resource,
pt.Attributes(), pt.Attributes(),
scope,
settings, settings,
nil, nil,
true, true,
@ -252,6 +254,7 @@ func convertBucketsLayout(bucketCounts []uint64, offset, scaleDown int32, adjust
func (c *PrometheusConverter) addCustomBucketsHistogramDataPoints(ctx context.Context, dataPoints pmetric.HistogramDataPointSlice, func (c *PrometheusConverter) addCustomBucketsHistogramDataPoints(ctx context.Context, dataPoints pmetric.HistogramDataPointSlice,
resource pcommon.Resource, settings Settings, promName string, temporality pmetric.AggregationTemporality, resource pcommon.Resource, settings Settings, promName string, temporality pmetric.AggregationTemporality,
scope scope,
) (annotations.Annotations, error) { ) (annotations.Annotations, error) {
var annots annotations.Annotations var annots annotations.Annotations
@ -271,6 +274,7 @@ func (c *PrometheusConverter) addCustomBucketsHistogramDataPoints(ctx context.Co
lbls := createAttributes( lbls := createAttributes(
resource, resource,
pt.Attributes(), pt.Attributes(),
scope,
settings, settings,
nil, nil,
true, true,

192
storage/remote/otlptranslator/prometheusremotewrite/histograms_test.go
View File

@ -620,13 +620,26 @@ func validateNativeHistogramCount(t *testing.T, h prompb.Histogram) {
} }
func TestPrometheusConverter_addExponentialHistogramDataPoints(t *testing.T) { func TestPrometheusConverter_addExponentialHistogramDataPoints(t *testing.T) {
scopeAttrs := pcommon.NewMap()
scopeAttrs.FromRaw(map[string]any{
"attr1": "value1",
"attr2": "value2",
})
defaultScope := scope{
name: "test-scope",
version: "1.0.0",
schemaURL: "https://schema.com",
attributes: scopeAttrs,
}
tests := []struct { tests := []struct {
name string name string
metric func() pmetric.Metric metric func() pmetric.Metric
wantSeries func() map[uint64]*prompb.TimeSeries scope scope
promoteScope bool
wantSeries func() map[uint64]*prompb.TimeSeries
}{ }{
{ {
name: "histogram data points with same labels", name: "histogram data points with same labels and without scope promotion",
metric: func() pmetric.Metric { metric: func() pmetric.Metric {
metric := pmetric.NewMetric() metric := pmetric.NewMetric()
metric.SetName("test_hist") metric.SetName("test_hist")
@ -650,6 +663,8 @@ func TestPrometheusConverter_addExponentialHistogramDataPoints(t *testing.T) {
return metric return metric
}, },
scope: defaultScope,
promoteScope: false,
wantSeries: func() map[uint64]*prompb.TimeSeries { wantSeries: func() map[uint64]*prompb.TimeSeries {
labels := []prompb.Label{ labels := []prompb.Label{
{Name: model.MetricNameLabel, Value: "test_hist"}, {Name: model.MetricNameLabel, Value: "test_hist"},
@ -685,7 +700,73 @@ func TestPrometheusConverter_addExponentialHistogramDataPoints(t *testing.T) {
}, },
}, },
{ {
name: "histogram data points with different labels", name: "histogram data points with same labels",
metric: func() pmetric.Metric {
metric := pmetric.NewMetric()
metric.SetName("test_hist")
metric.SetEmptyExponentialHistogram().SetAggregationTemporality(pmetric.AggregationTemporalityCumulative)
pt := metric.ExponentialHistogram().DataPoints().AppendEmpty()
pt.SetCount(7)
pt.SetScale(1)
pt.Positive().SetOffset(-1)
pt.Positive().BucketCounts().FromRaw([]uint64{4, 2})
pt.Exemplars().AppendEmpty().SetDoubleValue(1)
pt.Attributes().PutStr("attr", "test_attr")
pt = metric.ExponentialHistogram().DataPoints().AppendEmpty()
pt.SetCount(4)
pt.SetScale(1)
pt.Positive().SetOffset(-1)
pt.Positive().BucketCounts().FromRaw([]uint64{4, 2, 1})
pt.Exemplars().AppendEmpty().SetDoubleValue(2)
pt.Attributes().PutStr("attr", "test_attr")
return metric
},
scope: defaultScope,
promoteScope: true,
wantSeries: func() map[uint64]*prompb.TimeSeries {
labels := []prompb.Label{
{Name: model.MetricNameLabel, Value: "test_hist"},
{Name: "attr", Value: "test_attr"},
{Name: "otel_scope_name", Value: defaultScope.name},
{Name: "otel_scope_schema_url", Value: defaultScope.schemaURL},
{Name: "otel_scope_version", Value: defaultScope.version},
{Name: "otel_scope_attr1", Value: "value1"},
{Name: "otel_scope_attr2", Value: "value2"},
}
return map[uint64]*prompb.TimeSeries{
timeSeriesSignature(labels): {
Labels: labels,
Histograms: []prompb.Histogram{
{
Count: &prompb.Histogram_CountInt{CountInt: 7},
Schema: 1,
ZeroThreshold: defaultZeroThreshold,
ZeroCount: &prompb.Histogram_ZeroCountInt{ZeroCountInt: 0},
PositiveSpans: []prompb.BucketSpan{{Offset: 0, Length: 2}},
PositiveDeltas: []int64{4, -2},
},
{
Count: &prompb.Histogram_CountInt{CountInt: 4},
Schema: 1,
ZeroThreshold: defaultZeroThreshold,
ZeroCount: &prompb.Histogram_ZeroCountInt{ZeroCountInt: 0},
PositiveSpans: []prompb.BucketSpan{{Offset: 0, Length: 3}},
PositiveDeltas: []int64{4, -2, -1},
},
},
Exemplars: []prompb.Exemplar{
{Value: 1},
{Value: 2},
},
},
}
},
},
{
name: "histogram data points with different labels and without scope promotion",
metric: func() pmetric.Metric { metric: func() pmetric.Metric {
metric := pmetric.NewMetric() metric := pmetric.NewMetric()
metric.SetName("test_hist") metric.SetName("test_hist")
@ -709,6 +790,8 @@ func TestPrometheusConverter_addExponentialHistogramDataPoints(t *testing.T) {
return metric return metric
}, },
scope: defaultScope,
promoteScope: false,
wantSeries: func() map[uint64]*prompb.TimeSeries { wantSeries: func() map[uint64]*prompb.TimeSeries {
labels := []prompb.Label{ labels := []prompb.Label{
{Name: model.MetricNameLabel, Value: "test_hist"}, {Name: model.MetricNameLabel, Value: "test_hist"},
@ -769,10 +852,12 @@ func TestPrometheusConverter_addExponentialHistogramDataPoints(t *testing.T) {
metric.ExponentialHistogram().DataPoints(), metric.ExponentialHistogram().DataPoints(),
pcommon.NewResource(), pcommon.NewResource(),
Settings{ Settings{
ExportCreatedMetric: true, ExportCreatedMetric: true,
PromoteScopeMetadata: tt.promoteScope,
}, },
namer.Build(TranslatorMetricFromOtelMetric(metric)), namer.Build(TranslatorMetricFromOtelMetric(metric)),
pmetric.AggregationTemporalityCumulative, pmetric.AggregationTemporalityCumulative,
tt.scope,
) )
require.NoError(t, err) require.NoError(t, err)
require.Empty(t, annots) require.Empty(t, annots)
@ -991,13 +1076,26 @@ func TestHistogramToCustomBucketsHistogram(t *testing.T) {
} }
func TestPrometheusConverter_addCustomBucketsHistogramDataPoints(t *testing.T) { func TestPrometheusConverter_addCustomBucketsHistogramDataPoints(t *testing.T) {
scopeAttrs := pcommon.NewMap()
scopeAttrs.FromRaw(map[string]any{
"attr1": "value1",
"attr2": "value2",
})
defaultScope := scope{
name: "test-scope",
version: "1.0.0",
schemaURL: "https://schema.com",
attributes: scopeAttrs,
}
tests := []struct { tests := []struct {
name string name string
metric func() pmetric.Metric metric func() pmetric.Metric
wantSeries func() map[uint64]*prompb.TimeSeries scope scope
promoteScope bool
wantSeries func() map[uint64]*prompb.TimeSeries
}{ }{
{ {
name: "histogram data points with same labels", name: "histogram data points with same labels and without scope promotion",
metric: func() pmetric.Metric { metric: func() pmetric.Metric {
metric := pmetric.NewMetric() metric := pmetric.NewMetric()
metric.SetName("test_hist_to_nhcb") metric.SetName("test_hist_to_nhcb")
@ -1021,6 +1119,8 @@ func TestPrometheusConverter_addCustomBucketsHistogramDataPoints(t *testing.T) {
return metric return metric
}, },
scope: defaultScope,
promoteScope: false,
wantSeries: func() map[uint64]*prompb.TimeSeries { wantSeries: func() map[uint64]*prompb.TimeSeries {
labels := []prompb.Label{ labels := []prompb.Label{
{Name: model.MetricNameLabel, Value: "test_hist_to_nhcb"}, {Name: model.MetricNameLabel, Value: "test_hist_to_nhcb"},
@ -1056,7 +1156,73 @@ func TestPrometheusConverter_addCustomBucketsHistogramDataPoints(t *testing.T) {
}, },
}, },
{ {
name: "histogram data points with different labels", name: "histogram data points with same labels",
metric: func() pmetric.Metric {
metric := pmetric.NewMetric()
metric.SetName("test_hist_to_nhcb")
metric.SetEmptyHistogram().SetAggregationTemporality(pmetric.AggregationTemporalityCumulative)
pt := metric.Histogram().DataPoints().AppendEmpty()
pt.SetCount(3)
pt.SetSum(3)
pt.BucketCounts().FromRaw([]uint64{2, 0, 1})
pt.ExplicitBounds().FromRaw([]float64{5, 10})
pt.Exemplars().AppendEmpty().SetDoubleValue(1)
pt.Attributes().PutStr("attr", "test_attr")
pt = metric.Histogram().DataPoints().AppendEmpty()
pt.SetCount(11)
pt.SetSum(5)
pt.BucketCounts().FromRaw([]uint64{3, 8, 0})
pt.ExplicitBounds().FromRaw([]float64{0, 1})
pt.Exemplars().AppendEmpty().SetDoubleValue(2)
pt.Attributes().PutStr("attr", "test_attr")
return metric
},
scope: defaultScope,
promoteScope: true,
wantSeries: func() map[uint64]*prompb.TimeSeries {
labels := []prompb.Label{
{Name: model.MetricNameLabel, Value: "test_hist_to_nhcb"},
{Name: "attr", Value: "test_attr"},
{Name: "otel_scope_name", Value: defaultScope.name},
{Name: "otel_scope_schema_url", Value: defaultScope.schemaURL},
{Name: "otel_scope_version", Value: defaultScope.version},
{Name: "otel_scope_attr1", Value: "value1"},
{Name: "otel_scope_attr2", Value: "value2"},
}
return map[uint64]*prompb.TimeSeries{
timeSeriesSignature(labels): {
Labels: labels,
Histograms: []prompb.Histogram{
{
Count: &prompb.Histogram_CountInt{CountInt: 3},
Sum: 3,
Schema: -53,
PositiveSpans: []prompb.BucketSpan{{Offset: 0, Length: 3}},
PositiveDeltas: []int64{2, -2, 1},
CustomValues: []float64{5, 10},
},
{
Count: &prompb.Histogram_CountInt{CountInt: 11},
Sum: 5,
Schema: -53,
PositiveSpans: []prompb.BucketSpan{{Offset: 0, Length: 3}},
PositiveDeltas: []int64{3, 5, -8},
CustomValues: []float64{0, 1},
},
},
Exemplars: []prompb.Exemplar{
{Value: 1},
{Value: 2},
},
},
}
},
},
{
name: "histogram data points with different labels and without scope promotion",
metric: func() pmetric.Metric { metric: func() pmetric.Metric {
metric := pmetric.NewMetric() metric := pmetric.NewMetric()
metric.SetName("test_hist_to_nhcb") metric.SetName("test_hist_to_nhcb")
@ -1080,6 +1246,8 @@ func TestPrometheusConverter_addCustomBucketsHistogramDataPoints(t *testing.T) {
return metric return metric
}, },
scope: defaultScope,
promoteScope: false,
wantSeries: func() map[uint64]*prompb.TimeSeries { wantSeries: func() map[uint64]*prompb.TimeSeries {
labels := []prompb.Label{ labels := []prompb.Label{
{Name: model.MetricNameLabel, Value: "test_hist_to_nhcb"}, {Name: model.MetricNameLabel, Value: "test_hist_to_nhcb"},
@ -1142,9 +1310,11 @@ func TestPrometheusConverter_addCustomBucketsHistogramDataPoints(t *testing.T) {
Settings{ Settings{
ExportCreatedMetric: true, ExportCreatedMetric: true,
ConvertHistogramsToNHCB: true, ConvertHistogramsToNHCB: true,
PromoteScopeMetadata: tt.promoteScope,
}, },
namer.Build(TranslatorMetricFromOtelMetric(metric)), namer.Build(TranslatorMetricFromOtelMetric(metric)),
pmetric.AggregationTemporalityCumulative, pmetric.AggregationTemporalityCumulative,
tt.scope,
) )
require.NoError(t, err) require.NoError(t, err)

34
storage/remote/otlptranslator/prometheusremotewrite/metrics_to_prw.go
View File

@ -52,6 +52,8 @@ type Settings struct {
AllowDeltaTemporality bool AllowDeltaTemporality bool
// LookbackDelta is the PromQL engine lookback delta. // LookbackDelta is the PromQL engine lookback delta.
LookbackDelta time.Duration LookbackDelta time.Duration
// PromoteScopeMetadata controls whether to promote OTel scope metadata to metric labels.
PromoteScopeMetadata bool
} }
// PrometheusConverter converts from OTel write format to Prometheus remote write format. // PrometheusConverter converts from OTel write format to Prometheus remote write format.
@ -94,6 +96,23 @@ func TranslatorMetricFromOtelMetric(metric pmetric.Metric) otlptranslator.Metric
return m return m
} }
type scope struct {
name string
version string
schemaURL string
attributes pcommon.Map
}
func newScopeFromScopeMetrics(scopeMetrics pmetric.ScopeMetrics) scope {
s := scopeMetrics.Scope()
return scope{
name: s.Name(),
version: s.Version(),
schemaURL: scopeMetrics.SchemaUrl(),
attributes: s.Attributes(),
}
}
// FromMetrics converts pmetric.Metrics to Prometheus remote write format. // FromMetrics converts pmetric.Metrics to Prometheus remote write format.
func (c *PrometheusConverter) FromMetrics(ctx context.Context, md pmetric.Metrics, settings Settings) (annots annotations.Annotations, errs error) { func (c *PrometheusConverter) FromMetrics(ctx context.Context, md pmetric.Metrics, settings Settings) (annots annotations.Annotations, errs error) {
namer := otlptranslator.MetricNamer{ namer := otlptranslator.MetricNamer{
@ -122,7 +141,9 @@ func (c *PrometheusConverter) FromMetrics(ctx context.Context, md pmetric.Metric
earliestTimestamp := pcommon.Timestamp(math.MaxUint64) earliestTimestamp := pcommon.Timestamp(math.MaxUint64)
latestTimestamp := pcommon.Timestamp(0) latestTimestamp := pcommon.Timestamp(0)
for j := 0; j < scopeMetricsSlice.Len(); j++ { for j := 0; j < scopeMetricsSlice.Len(); j++ {
metricSlice := scopeMetricsSlice.At(j).Metrics() scopeMetrics := scopeMetricsSlice.At(j)
scope := newScopeFromScopeMetrics(scopeMetrics)
metricSlice := scopeMetrics.Metrics()
// TODO: decide if instrumentation library information should be exported as labels // TODO: decide if instrumentation library information should be exported as labels
for k := 0; k < metricSlice.Len(); k++ { for k := 0; k < metricSlice.Len(); k++ {
@ -166,7 +187,7 @@ func (c *PrometheusConverter) FromMetrics(ctx context.Context, md pmetric.Metric
errs = multierr.Append(errs, fmt.Errorf("empty data points. %s is dropped", metric.Name())) errs = multierr.Append(errs, fmt.Errorf("empty data points. %s is dropped", metric.Name()))
break break
} }
if err := c.addGaugeNumberDataPoints(ctx, dataPoints, resource, settings, promName); err != nil { if err := c.addGaugeNumberDataPoints(ctx, dataPoints, resource, settings, promName, scope); err != nil {
errs = multierr.Append(errs, err) errs = multierr.Append(errs, err)
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return return
@ -178,7 +199,7 @@ func (c *PrometheusConverter) FromMetrics(ctx context.Context, md pmetric.Metric
errs = multierr.Append(errs, fmt.Errorf("empty data points. %s is dropped", metric.Name())) errs = multierr.Append(errs, fmt.Errorf("empty data points. %s is dropped", metric.Name()))
break break
} }
if err := c.addSumNumberDataPoints(ctx, dataPoints, resource, metric, settings, promName); err != nil { if err := c.addSumNumberDataPoints(ctx, dataPoints, resource, metric, settings, promName, scope); err != nil {
errs = multierr.Append(errs, err) errs = multierr.Append(errs, err)
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return return
@ -192,7 +213,7 @@ func (c *PrometheusConverter) FromMetrics(ctx context.Context, md pmetric.Metric
} }
if settings.ConvertHistogramsToNHCB { if settings.ConvertHistogramsToNHCB {
ws, err := c.addCustomBucketsHistogramDataPoints( ws, err := c.addCustomBucketsHistogramDataPoints(
ctx, dataPoints, resource, settings, promName, temporality, ctx, dataPoints, resource, settings, promName, temporality, scope,
) )
annots.Merge(ws) annots.Merge(ws)
if err != nil { if err != nil {
@ -202,7 +223,7 @@ func (c *PrometheusConverter) FromMetrics(ctx context.Context, md pmetric.Metric
} }
} }
} else { } else {
if err := c.addHistogramDataPoints(ctx, dataPoints, resource, settings, promName); err != nil { if err := c.addHistogramDataPoints(ctx, dataPoints, resource, settings, promName, scope); err != nil {
errs = multierr.Append(errs, err) errs = multierr.Append(errs, err)
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return return
@ -222,6 +243,7 @@ func (c *PrometheusConverter) FromMetrics(ctx context.Context, md pmetric.Metric
settings, settings,
promName, promName,
temporality, temporality,
scope,
) )
annots.Merge(ws) annots.Merge(ws)
if err != nil { if err != nil {
@ -236,7 +258,7 @@ func (c *PrometheusConverter) FromMetrics(ctx context.Context, md pmetric.Metric
errs = multierr.Append(errs, fmt.Errorf("empty data points. %s is dropped", metric.Name())) errs = multierr.Append(errs, fmt.Errorf("empty data points. %s is dropped", metric.Name()))
break break
} }
if err := c.addSummaryDataPoints(ctx, dataPoints, resource, settings, promName); err != nil { if err := c.addSummaryDataPoints(ctx, dataPoints, resource, settings, promName, scope); err != nil {
errs = multierr.Append(errs, err) errs = multierr.Append(errs, err)
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return return

6
storage/remote/otlptranslator/prometheusremotewrite/number_data_points.go
View File

@ -29,7 +29,7 @@ import (
) )
func (c *PrometheusConverter) addGaugeNumberDataPoints(ctx context.Context, dataPoints pmetric.NumberDataPointSlice, func (c *PrometheusConverter) addGaugeNumberDataPoints(ctx context.Context, dataPoints pmetric.NumberDataPointSlice,
resource pcommon.Resource, settings Settings, name string, resource pcommon.Resource, settings Settings, name string, scope scope,
) error { ) error {
for x := 0; x < dataPoints.Len(); x++ { for x := 0; x < dataPoints.Len(); x++ {
if err := c.everyN.checkContext(ctx); err != nil { if err := c.everyN.checkContext(ctx); err != nil {
@ -40,6 +40,7 @@ func (c *PrometheusConverter) addGaugeNumberDataPoints(ctx context.Context, data
labels := createAttributes( labels := createAttributes(
resource, resource,
pt.Attributes(), pt.Attributes(),
scope,
settings, settings,
nil, nil,
true, true,
@ -66,7 +67,7 @@ func (c *PrometheusConverter) addGaugeNumberDataPoints(ctx context.Context, data
} }
func (c *PrometheusConverter) addSumNumberDataPoints(ctx context.Context, dataPoints pmetric.NumberDataPointSlice, func (c *PrometheusConverter) addSumNumberDataPoints(ctx context.Context, dataPoints pmetric.NumberDataPointSlice,
resource pcommon.Resource, metric pmetric.Metric, settings Settings, name string, resource pcommon.Resource, metric pmetric.Metric, settings Settings, name string, scope scope,
) error { ) error {
for x := 0; x < dataPoints.Len(); x++ { for x := 0; x < dataPoints.Len(); x++ {
if err := c.everyN.checkContext(ctx); err != nil { if err := c.everyN.checkContext(ctx); err != nil {
@ -77,6 +78,7 @@ func (c *PrometheusConverter) addSumNumberDataPoints(ctx context.Context, dataPo
lbls := createAttributes( lbls := createAttributes(
resource, resource,
pt.Attributes(), pt.Attributes(),
scope,
settings, settings,
nil, nil,
true, true,

137
storage/remote/otlptranslator/prometheusremotewrite/number_data_points_test.go
View File

@ -30,14 +30,27 @@ import (
) )
func TestPrometheusConverter_addGaugeNumberDataPoints(t *testing.T) { func TestPrometheusConverter_addGaugeNumberDataPoints(t *testing.T) {
scopeAttrs := pcommon.NewMap()
scopeAttrs.FromRaw(map[string]any{
"attr1": "value1",
"attr2": "value2",
})
defaultScope := scope{
name: "test-scope",
version: "1.0.0",
schemaURL: "https://schema.com",
attributes: scopeAttrs,
}
ts := uint64(time.Now().UnixNano()) ts := uint64(time.Now().UnixNano())
tests := []struct { tests := []struct {
name string name string
metric func() pmetric.Metric metric func() pmetric.Metric
want func() map[uint64]*prompb.TimeSeries scope scope
promoteScope bool
want func() map[uint64]*prompb.TimeSeries
}{ }{
{ {
name: "gauge", name: "gauge without scope promotion",
metric: func() pmetric.Metric { metric: func() pmetric.Metric {
return getIntGaugeMetric( return getIntGaugeMetric(
"test", "test",
@ -45,6 +58,8 @@ func TestPrometheusConverter_addGaugeNumberDataPoints(t *testing.T) {
1, ts, 1, ts,
) )
}, },
scope: defaultScope,
promoteScope: false,
want: func() map[uint64]*prompb.TimeSeries { want: func() map[uint64]*prompb.TimeSeries {
labels := []prompb.Label{ labels := []prompb.Label{
{Name: model.MetricNameLabel, Value: "test"}, {Name: model.MetricNameLabel, Value: "test"},
@ -62,6 +77,39 @@ func TestPrometheusConverter_addGaugeNumberDataPoints(t *testing.T) {
} }
}, },
}, },
{
name: "gauge with scope promotion",
metric: func() pmetric.Metric {
return getIntGaugeMetric(
"test",
pcommon.NewMap(),
1, ts,
)
},
scope: defaultScope,
promoteScope: true,
want: func() map[uint64]*prompb.TimeSeries {
labels := []prompb.Label{
{Name: model.MetricNameLabel, Value: "test"},
{Name: "otel_scope_name", Value: defaultScope.name},
{Name: "otel_scope_schema_url", Value: defaultScope.schemaURL},
{Name: "otel_scope_version", Value: defaultScope.version},
{Name: "otel_scope_attr1", Value: "value1"},
{Name: "otel_scope_attr2", Value: "value2"},
}
return map[uint64]*prompb.TimeSeries{
timeSeriesSignature(labels): {
Labels: labels,
Samples: []prompb.Sample{
{
Value: 1,
Timestamp: convertTimeStamp(pcommon.Timestamp(ts)),
},
},
},
}
},
},
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
@ -73,9 +121,11 @@ func TestPrometheusConverter_addGaugeNumberDataPoints(t *testing.T) {
metric.Gauge().DataPoints(), metric.Gauge().DataPoints(),
pcommon.NewResource(), pcommon.NewResource(),
Settings{ Settings{
ExportCreatedMetric: true, ExportCreatedMetric: true,
PromoteScopeMetadata: tt.promoteScope,
}, },
metric.Name(), metric.Name(),
tt.scope,
) )
require.Equal(t, tt.want(), converter.unique) require.Equal(t, tt.want(), converter.unique)
@ -85,14 +135,27 @@ func TestPrometheusConverter_addGaugeNumberDataPoints(t *testing.T) {
} }
func TestPrometheusConverter_addSumNumberDataPoints(t *testing.T) { func TestPrometheusConverter_addSumNumberDataPoints(t *testing.T) {
scopeAttrs := pcommon.NewMap()
scopeAttrs.FromRaw(map[string]any{
"attr1": "value1",
"attr2": "value2",
})
defaultScope := scope{
name: "test-scope",
version: "1.0.0",
schemaURL: "https://schema.com",
attributes: scopeAttrs,
}
ts := pcommon.Timestamp(time.Now().UnixNano()) ts := pcommon.Timestamp(time.Now().UnixNano())
tests := []struct { tests := []struct {
name string name string
metric func() pmetric.Metric metric func() pmetric.Metric
want func() map[uint64]*prompb.TimeSeries scope scope
promoteScope bool
want func() map[uint64]*prompb.TimeSeries
}{ }{
{ {
name: "sum", name: "sum without scope promotion",
metric: func() pmetric.Metric { metric: func() pmetric.Metric {
return getIntSumMetric( return getIntSumMetric(
"test", "test",
@ -101,6 +164,8 @@ func TestPrometheusConverter_addSumNumberDataPoints(t *testing.T) {
uint64(ts.AsTime().UnixNano()), uint64(ts.AsTime().UnixNano()),
) )
}, },
scope: defaultScope,
promoteScope: false,
want: func() map[uint64]*prompb.TimeSeries { want: func() map[uint64]*prompb.TimeSeries {
labels := []prompb.Label{ labels := []prompb.Label{
{Name: model.MetricNameLabel, Value: "test"}, {Name: model.MetricNameLabel, Value: "test"},
@ -119,7 +184,41 @@ func TestPrometheusConverter_addSumNumberDataPoints(t *testing.T) {
}, },
}, },
{ {
name: "sum with exemplars", name: "sum with scope promotion",
metric: func() pmetric.Metric {
return getIntSumMetric(
"test",
pcommon.NewMap(),
1,
uint64(ts.AsTime().UnixNano()),
)
},
scope: defaultScope,
promoteScope: true,
want: func() map[uint64]*prompb.TimeSeries {
labels := []prompb.Label{
{Name: model.MetricNameLabel, Value: "test"},
{Name: "otel_scope_name", Value: defaultScope.name},
{Name: "otel_scope_schema_url", Value: defaultScope.schemaURL},
{Name: "otel_scope_version", Value: defaultScope.version},
{Name: "otel_scope_attr1", Value: "value1"},
{Name: "otel_scope_attr2", Value: "value2"},
}
return map[uint64]*prompb.TimeSeries{
timeSeriesSignature(labels): {
Labels: labels,
Samples: []prompb.Sample{
{
Value: 1,
Timestamp: convertTimeStamp(ts),
},
},
},
}
},
},
{
name: "sum with exemplars and without scope promotion",
metric: func() pmetric.Metric { metric: func() pmetric.Metric {
m := getIntSumMetric( m := getIntSumMetric(
"test", "test",
@ -130,6 +229,8 @@ func TestPrometheusConverter_addSumNumberDataPoints(t *testing.T) {
m.Sum().DataPoints().At(0).Exemplars().AppendEmpty().SetDoubleValue(2) m.Sum().DataPoints().At(0).Exemplars().AppendEmpty().SetDoubleValue(2)
return m return m
}, },
scope: defaultScope,
promoteScope: false,
want: func() map[uint64]*prompb.TimeSeries { want: func() map[uint64]*prompb.TimeSeries {
labels := []prompb.Label{ labels := []prompb.Label{
{Name: model.MetricNameLabel, Value: "test"}, {Name: model.MetricNameLabel, Value: "test"},
@ -149,7 +250,7 @@ func TestPrometheusConverter_addSumNumberDataPoints(t *testing.T) {
}, },
}, },
{ {
name: "monotonic cumulative sum with start timestamp", name: "monotonic cumulative sum with start timestamp and without scope promotion",
metric: func() pmetric.Metric { metric: func() pmetric.Metric {
metric := pmetric.NewMetric() metric := pmetric.NewMetric()
metric.SetName("test_sum") metric.SetName("test_sum")
@ -163,6 +264,8 @@ func TestPrometheusConverter_addSumNumberDataPoints(t *testing.T) {
return metric return metric
}, },
scope: defaultScope,
promoteScope: false,
want: func() map[uint64]*prompb.TimeSeries { want: func() map[uint64]*prompb.TimeSeries {
labels := []prompb.Label{ labels := []prompb.Label{
{Name: model.MetricNameLabel, Value: "test_sum"}, {Name: model.MetricNameLabel, Value: "test_sum"},
@ -187,7 +290,7 @@ func TestPrometheusConverter_addSumNumberDataPoints(t *testing.T) {
}, },
}, },
{ {
name: "monotonic cumulative sum with no start time", name: "monotonic cumulative sum with no start time and without scope promotion",
metric: func() pmetric.Metric { metric: func() pmetric.Metric {
metric := pmetric.NewMetric() metric := pmetric.NewMetric()
metric.SetName("test_sum") metric.SetName("test_sum")
@ -199,6 +302,8 @@ func TestPrometheusConverter_addSumNumberDataPoints(t *testing.T) {
return metric return metric
}, },
scope: defaultScope,
promoteScope: false,
want: func() map[uint64]*prompb.TimeSeries { want: func() map[uint64]*prompb.TimeSeries {
labels := []prompb.Label{ labels := []prompb.Label{
{Name: model.MetricNameLabel, Value: "test_sum"}, {Name: model.MetricNameLabel, Value: "test_sum"},
@ -214,7 +319,7 @@ func TestPrometheusConverter_addSumNumberDataPoints(t *testing.T) {
}, },
}, },
{ {
name: "non-monotonic cumulative sum with start time", name: "non-monotonic cumulative sum with start time and without scope promotion",
metric: func() pmetric.Metric { metric: func() pmetric.Metric {
metric := pmetric.NewMetric() metric := pmetric.NewMetric()
metric.SetName("test_sum") metric.SetName("test_sum")
@ -226,6 +331,8 @@ func TestPrometheusConverter_addSumNumberDataPoints(t *testing.T) {
return metric return metric
}, },
scope: defaultScope,
promoteScope: false,
want: func() map[uint64]*prompb.TimeSeries { want: func() map[uint64]*prompb.TimeSeries {
labels := []prompb.Label{ labels := []prompb.Label{
{Name: model.MetricNameLabel, Value: "test_sum"}, {Name: model.MetricNameLabel, Value: "test_sum"},
@ -252,9 +359,11 @@ func TestPrometheusConverter_addSumNumberDataPoints(t *testing.T) {
pcommon.NewResource(), pcommon.NewResource(),
metric, metric,
Settings{ Settings{
ExportCreatedMetric: true, ExportCreatedMetric: true,
PromoteScopeMetadata: tt.promoteScope,
}, },
metric.Name(), metric.Name(),
tt.scope,
) )
require.Equal(t, tt.want(), converter.unique) require.Equal(t, tt.want(), converter.unique)

1
storage/remote/write_handler.go
View File

@ -603,6 +603,7 @@ func (rw *rwExporter) ConsumeMetrics(ctx context.Context, md pmetric.Metrics) er
ConvertHistogramsToNHCB: otlpCfg.ConvertHistogramsToNHCB, ConvertHistogramsToNHCB: otlpCfg.ConvertHistogramsToNHCB,
AllowDeltaTemporality: rw.allowDeltaTemporality, AllowDeltaTemporality: rw.allowDeltaTemporality,
LookbackDelta: rw.lookbackDelta, LookbackDelta: rw.lookbackDelta,
PromoteScopeMetadata: otlpCfg.PromoteScopeMetadata,
}) })
if err != nil { if err != nil {
rw.logger.Warn("Error translating OTLP metrics to Prometheus write request", "err", err) rw.logger.Warn("Error translating OTLP metrics to Prometheus write request", "err", err)