From 2834a665ed6460e2f92a0f94eba07ceed2f096d4 Mon Sep 17 00:00:00 2001 From: Antonio Jimenez <123171955+antonjim-te@users.noreply.github.com> Date: Mon, 26 May 2025 18:15:01 +0200 Subject: [PATCH] Add support for promoting all OTel resource attributes (#16426) Add support for promoting all OTel resource attributes via `promote_all_resource_attributes`, except for those ignored using 'ignore_resource_attributes'. --------- Signed-off-by: Antonio Jimenez Signed-off-by: Antonio Jimenez <123171955+antonjim-te@users.noreply.github.com> --- config/config.go | 30 +++++- config/config_test.go | 68 +++++++++++++- ...rce_attributes_without_promote_all.bad.yml | 2 + ...lp_promote_all_resource_attributes.bad.yml | 3 + ...itize_default_resource_attributes.good.yml | 1 + ...anitize_ignore_resource_attributes.bad.yml | 3 + ...nitize_ignore_resource_attributes.good.yml | 3 + ...itize_promote_resource_attributes.bad.yml} | 0 ...tize_promote_resource_attributes.good.yml} | 0 ...e_resource_attributes_promote_all.good.yml | 2 + docs/configuration/configuration.md | 8 ++ .../prometheusremotewrite/helper.go | 8 +- .../prometheusremotewrite/helper_test.go | 92 ++++++++++++++++++- .../prometheusremotewrite/metrics_to_prw.go | 51 +++++++++- storage/remote/write_handler.go | 2 +- 15 files changed, 251 insertions(+), 22 deletions(-) create mode 100644 config/testdata/otlp_ignore_resource_attributes_without_promote_all.bad.yml create mode 100644 config/testdata/otlp_promote_all_resource_attributes.bad.yml create mode 100644 config/testdata/otlp_sanitize_default_resource_attributes.good.yml create mode 100644 config/testdata/otlp_sanitize_ignore_resource_attributes.bad.yml create mode 100644 config/testdata/otlp_sanitize_ignore_resource_attributes.good.yml rename config/testdata/{otlp_sanitize_resource_attributes.bad.yml => otlp_sanitize_promote_resource_attributes.bad.yml} (100%) rename config/testdata/{otlp_sanitize_resource_attributes.good.yml => otlp_sanitize_promote_resource_attributes.good.yml} (100%) create mode 100644 config/testdata/otlp_sanitize_resource_attributes_promote_all.good.yml diff --git a/config/config.go b/config/config.go index fcf30f6cad..0eb618d5b1 100644 --- a/config/config.go +++ b/config/config.go @@ -1555,7 +1555,9 @@ var ( // OTLPConfig is the configuration for writing to the OTLP endpoint. type OTLPConfig struct { + PromoteAllResourceAttributes bool `yaml:"promote_all_resource_attributes,omitempty"` PromoteResourceAttributes []string `yaml:"promote_resource_attributes,omitempty"` + IgnoreResourceAttributes []string `yaml:"ignore_resource_attributes,omitempty"` TranslationStrategy translationStrategyOption `yaml:"translation_strategy,omitempty"` KeepIdentifyingResourceAttributes bool `yaml:"keep_identifying_resource_attributes,omitempty"` ConvertHistogramsToNHCB bool `yaml:"convert_histograms_to_nhcb,omitempty"` @@ -1569,21 +1571,41 @@ func (c *OTLPConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { return err } + if c.PromoteAllResourceAttributes { + if len(c.PromoteResourceAttributes) > 0 { + return errors.New("'promote_all_resource_attributes' and 'promote_resource_attributes' cannot be configured simultaneously") + } + if err := sanitizeAttributes(c.IgnoreResourceAttributes, "ignored"); err != nil { + return fmt.Errorf("invalid 'ignore_resource_attributes': %w", err) + } + } else { + if len(c.IgnoreResourceAttributes) > 0 { + return errors.New("'ignore_resource_attributes' cannot be configured unless 'promote_all_resource_attributes' is true") + } + if err := sanitizeAttributes(c.PromoteResourceAttributes, "promoted"); err != nil { + return fmt.Errorf("invalid 'promote_resource_attributes': %w", err) + } + } + + return nil +} + +func sanitizeAttributes(attributes []string, adjective string) error { seen := map[string]struct{}{} var err error - for i, attr := range c.PromoteResourceAttributes { + for i, attr := range attributes { attr = strings.TrimSpace(attr) if attr == "" { - err = errors.Join(err, errors.New("empty promoted OTel resource attribute")) + err = errors.Join(err, fmt.Errorf("empty %s OTel resource attribute", adjective)) continue } if _, exists := seen[attr]; exists { - err = errors.Join(err, fmt.Errorf("duplicated promoted OTel resource attribute %q", attr)) + err = errors.Join(err, fmt.Errorf("duplicated %s OTel resource attribute %q", adjective, attr)) continue } seen[attr] = struct{}{} - c.PromoteResourceAttributes[i] = attr + attributes[i] = attr } return err } diff --git a/config/config_test.go b/config/config_test.go index 1d3e4099c1..9735b18968 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -1661,8 +1661,8 @@ func TestRemoteWriteRetryOnRateLimit(t *testing.T) { } func TestOTLPSanitizeResourceAttributes(t *testing.T) { - t.Run("good config", func(t *testing.T) { - want, err := LoadFile(filepath.Join("testdata", "otlp_sanitize_resource_attributes.good.yml"), false, promslog.NewNopLogger()) + t.Run("good config - default resource attributes", func(t *testing.T) { + want, err := LoadFile(filepath.Join("testdata", "otlp_sanitize_default_resource_attributes.good.yml"), false, promslog.NewNopLogger()) require.NoError(t, err) out, err := yaml.Marshal(want) @@ -1670,14 +1670,74 @@ func TestOTLPSanitizeResourceAttributes(t *testing.T) { var got Config require.NoError(t, yaml.UnmarshalStrict(out, &got)) + require.False(t, got.OTLPConfig.PromoteAllResourceAttributes) + require.Empty(t, got.OTLPConfig.IgnoreResourceAttributes) + require.Empty(t, got.OTLPConfig.PromoteResourceAttributes) + }) + + t.Run("good config - promote resource attributes", func(t *testing.T) { + want, err := LoadFile(filepath.Join("testdata", "otlp_sanitize_promote_resource_attributes.good.yml"), false, promslog.NewNopLogger()) + require.NoError(t, err) + + out, err := yaml.Marshal(want) + require.NoError(t, err) + var got Config + require.NoError(t, yaml.UnmarshalStrict(out, &got)) + + require.False(t, got.OTLPConfig.PromoteAllResourceAttributes) + require.Empty(t, got.OTLPConfig.IgnoreResourceAttributes) require.Equal(t, []string{"k8s.cluster.name", "k8s.job.name", "k8s.namespace.name"}, got.OTLPConfig.PromoteResourceAttributes) }) - t.Run("bad config", func(t *testing.T) { - _, err := LoadFile(filepath.Join("testdata", "otlp_sanitize_resource_attributes.bad.yml"), false, promslog.NewNopLogger()) + t.Run("bad config - promote resource attributes", func(t *testing.T) { + _, err := LoadFile(filepath.Join("testdata", "otlp_sanitize_promote_resource_attributes.bad.yml"), false, promslog.NewNopLogger()) + require.ErrorContains(t, err, `invalid 'promote_resource_attributes'`) require.ErrorContains(t, err, `duplicated promoted OTel resource attribute "k8s.job.name"`) require.ErrorContains(t, err, `empty promoted OTel resource attribute`) }) + + t.Run("good config - promote all resource attributes", func(t *testing.T) { + want, err := LoadFile(filepath.Join("testdata", "otlp_sanitize_resource_attributes_promote_all.good.yml"), false, promslog.NewNopLogger()) + require.NoError(t, err) + + out, err := yaml.Marshal(want) + require.NoError(t, err) + var got Config + require.NoError(t, yaml.UnmarshalStrict(out, &got)) + require.True(t, got.OTLPConfig.PromoteAllResourceAttributes) + require.Empty(t, got.OTLPConfig.PromoteResourceAttributes) + require.Empty(t, got.OTLPConfig.IgnoreResourceAttributes) + }) + + t.Run("good config - ignore resource attributes", func(t *testing.T) { + want, err := LoadFile(filepath.Join("testdata", "otlp_sanitize_ignore_resource_attributes.good.yml"), false, promslog.NewNopLogger()) + require.NoError(t, err) + + out, err := yaml.Marshal(want) + require.NoError(t, err) + var got Config + require.NoError(t, yaml.UnmarshalStrict(out, &got)) + require.True(t, got.OTLPConfig.PromoteAllResourceAttributes) + require.Empty(t, got.OTLPConfig.PromoteResourceAttributes) + require.Equal(t, []string{"k8s.cluster.name", "k8s.job.name", "k8s.namespace.name"}, got.OTLPConfig.IgnoreResourceAttributes) + }) + + t.Run("bad config - ignore resource attributes", func(t *testing.T) { + _, err := LoadFile(filepath.Join("testdata", "otlp_sanitize_ignore_resource_attributes.bad.yml"), false, promslog.NewNopLogger()) + require.ErrorContains(t, err, `invalid 'ignore_resource_attributes'`) + require.ErrorContains(t, err, `duplicated ignored OTel resource attribute "k8s.job.name"`) + require.ErrorContains(t, err, `empty ignored OTel resource attribute`) + }) + + t.Run("bad config - conflict between promote all and promote specific resource attributes", func(t *testing.T) { + _, err := LoadFile(filepath.Join("testdata", "otlp_promote_all_resource_attributes.bad.yml"), false, promslog.NewNopLogger()) + require.ErrorContains(t, err, `'promote_all_resource_attributes' and 'promote_resource_attributes' cannot be configured simultaneously`) + }) + + t.Run("bad config - configuring ignoring of resource attributes without also enabling promotion of all resource attributes", func(t *testing.T) { + _, err := LoadFile(filepath.Join("testdata", "otlp_ignore_resource_attributes_without_promote_all.bad.yml"), false, promslog.NewNopLogger()) + require.ErrorContains(t, err, `'ignore_resource_attributes' cannot be configured unless 'promote_all_resource_attributes' is true`) + }) } func TestOTLPAllowServiceNameInTargetInfo(t *testing.T) { diff --git a/config/testdata/otlp_ignore_resource_attributes_without_promote_all.bad.yml b/config/testdata/otlp_ignore_resource_attributes_without_promote_all.bad.yml new file mode 100644 index 0000000000..be4ee60f2a --- /dev/null +++ b/config/testdata/otlp_ignore_resource_attributes_without_promote_all.bad.yml @@ -0,0 +1,2 @@ +otlp: + ignore_resource_attributes: ["k8s.job.name"] diff --git a/config/testdata/otlp_promote_all_resource_attributes.bad.yml b/config/testdata/otlp_promote_all_resource_attributes.bad.yml new file mode 100644 index 0000000000..2be54ec155 --- /dev/null +++ b/config/testdata/otlp_promote_all_resource_attributes.bad.yml @@ -0,0 +1,3 @@ +otlp: + promote_all_resource_attributes: true + promote_resource_attributes: ["k8s.cluster.name", " k8s.job.name ", "k8s.namespace.name", "k8s.job.name"] diff --git a/config/testdata/otlp_sanitize_default_resource_attributes.good.yml b/config/testdata/otlp_sanitize_default_resource_attributes.good.yml new file mode 100644 index 0000000000..abdd98dc7a --- /dev/null +++ b/config/testdata/otlp_sanitize_default_resource_attributes.good.yml @@ -0,0 +1 @@ +otlp: diff --git a/config/testdata/otlp_sanitize_ignore_resource_attributes.bad.yml b/config/testdata/otlp_sanitize_ignore_resource_attributes.bad.yml new file mode 100644 index 0000000000..57ce0efac0 --- /dev/null +++ b/config/testdata/otlp_sanitize_ignore_resource_attributes.bad.yml @@ -0,0 +1,3 @@ +otlp: + promote_all_resource_attributes: true + ignore_resource_attributes: ["k8s.cluster.name", " k8s.job.name ", "k8s.namespace.name", "k8s.job.name", ""] diff --git a/config/testdata/otlp_sanitize_ignore_resource_attributes.good.yml b/config/testdata/otlp_sanitize_ignore_resource_attributes.good.yml new file mode 100644 index 0000000000..7a50fef405 --- /dev/null +++ b/config/testdata/otlp_sanitize_ignore_resource_attributes.good.yml @@ -0,0 +1,3 @@ +otlp: + promote_all_resource_attributes: true + ignore_resource_attributes: ["k8s.cluster.name", " k8s.job.name ", "k8s.namespace.name"] diff --git a/config/testdata/otlp_sanitize_resource_attributes.bad.yml b/config/testdata/otlp_sanitize_promote_resource_attributes.bad.yml similarity index 100% rename from config/testdata/otlp_sanitize_resource_attributes.bad.yml rename to config/testdata/otlp_sanitize_promote_resource_attributes.bad.yml diff --git a/config/testdata/otlp_sanitize_resource_attributes.good.yml b/config/testdata/otlp_sanitize_promote_resource_attributes.good.yml similarity index 100% rename from config/testdata/otlp_sanitize_resource_attributes.good.yml rename to config/testdata/otlp_sanitize_promote_resource_attributes.good.yml diff --git a/config/testdata/otlp_sanitize_resource_attributes_promote_all.good.yml b/config/testdata/otlp_sanitize_resource_attributes_promote_all.good.yml new file mode 100644 index 0000000000..2c8360011b --- /dev/null +++ b/config/testdata/otlp_sanitize_resource_attributes_promote_all.good.yml @@ -0,0 +1,2 @@ +otlp: + promote_all_resource_attributes: true diff --git a/docs/configuration/configuration.md b/docs/configuration/configuration.md index 413076e929..d1b1b752c3 100644 --- a/docs/configuration/configuration.md +++ b/docs/configuration/configuration.md @@ -183,7 +183,15 @@ remote_write: # Settings related to the OTLP receiver feature. # See https://prometheus.io/docs/guides/opentelemetry/ for best practices. otlp: + # Promote specific list of resource attributes to labels. + # It cannot be configured simultaneously with 'promote_all_resource_attributes: true'. [ promote_resource_attributes: [, ...] | default = [ ] ] + # Promoting all resource attributes to labels, except for the ones configured with 'ignore_resource_attributes'. + # Be aware that changes in attributes received by the OTLP endpoint may result in time series churn and lead to high memory usage by the Prometheus server. + # It cannot be set to 'true' simultaneously with 'promote_resource_attributes'. + [ promote_all_resource_attributes: | default = false ] + # Which resource attributes to ignore, can only be set when 'promote_all_resource_attributes' is true. + [ ignore_resource_attributes: [, ...] | default = [] ] # Configures translation of OTLP metrics when received through the OTLP metrics # endpoint. Available values: # - "UnderscoreEscapingWithSuffixes" refers to commonly agreed normalization used diff --git a/storage/remote/otlptranslator/prometheusremotewrite/helper.go b/storage/remote/otlptranslator/prometheusremotewrite/helper.go index 527a0c879f..9087c8d7f8 100644 --- a/storage/remote/otlptranslator/prometheusremotewrite/helper.go +++ b/storage/remote/otlptranslator/prometheusremotewrite/helper.go @@ -122,13 +122,7 @@ func createAttributes(resource pcommon.Resource, attributes pcommon.Map, setting serviceName, haveServiceName := resourceAttrs.Get(conventions.AttributeServiceName) instance, haveInstanceID := resourceAttrs.Get(conventions.AttributeServiceInstanceID) - promotedAttrs := make([]prompb.Label, 0, len(settings.PromoteResourceAttributes)) - for _, name := range settings.PromoteResourceAttributes { - if value, exists := resourceAttrs.Get(name); exists { - promotedAttrs = append(promotedAttrs, prompb.Label{Name: name, Value: value.AsString()}) - } - } - sort.Stable(ByLabelName(promotedAttrs)) + promotedAttrs := settings.PromoteResourceAttributes.promotedAttributes(resourceAttrs) // Calculate the maximum possible number of labels we could return so we can preallocate l maxLabelCount := attributes.Len() + len(settings.ExternalLabels) + len(promotedAttrs) + len(extras)/2 diff --git a/storage/remote/otlptranslator/prometheusremotewrite/helper_test.go b/storage/remote/otlptranslator/prometheusremotewrite/helper_test.go index bc12934091..578a3a6168 100644 --- a/storage/remote/otlptranslator/prometheusremotewrite/helper_test.go +++ b/storage/remote/otlptranslator/prometheusremotewrite/helper_test.go @@ -26,6 +26,7 @@ import ( "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/pmetric" + "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/prompb" ) @@ -51,10 +52,12 @@ func TestCreateAttributes(t *testing.T) { attrs.PutStr("metric-attr-other", "metric value other") testCases := []struct { - name string - promoteResourceAttributes []string - ignoreAttrs []string - expectedLabels []prompb.Label + name string + promoteAllResourceAttributes bool + promoteResourceAttributes []string + ignoreResourceAttributes []string + ignoreAttrs []string + expectedLabels []prompb.Label }{ { name: "Successful conversion without resource attribute promotion", @@ -195,11 +198,90 @@ func TestCreateAttributes(t *testing.T) { }, }, }, + { + name: "Successful conversion promoting all resource attributes", + promoteAllResourceAttributes: true, + expectedLabels: []prompb.Label{ + { + Name: "__name__", + Value: "test_metric", + }, + { + Name: "instance", + Value: "service ID", + }, + { + Name: "job", + Value: "service name", + }, + { + Name: "existent_attr", + Value: "resource value", + }, + { + Name: "metric_attr", + Value: "metric value", + }, + { + Name: "metric_attr_other", + Value: "metric value other", + }, + { + Name: "service_name", + Value: "service name", + }, + { + Name: "service_instance_id", + Value: "service ID", + }, + }, + }, + { + name: "Successful conversion promoting all resource attributes, ignoring 'service.instance.id'", + promoteAllResourceAttributes: true, + ignoreResourceAttributes: []string{ + "service.instance.id", + }, + expectedLabels: []prompb.Label{ + { + Name: "__name__", + Value: "test_metric", + }, + { + Name: "instance", + Value: "service ID", + }, + { + Name: "job", + Value: "service name", + }, + { + Name: "existent_attr", + Value: "resource value", + }, + { + Name: "metric_attr", + Value: "metric value", + }, + { + Name: "metric_attr_other", + Value: "metric value other", + }, + { + Name: "service_name", + Value: "service name", + }, + }, + }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { settings := Settings{ - PromoteResourceAttributes: tc.promoteResourceAttributes, + PromoteResourceAttributes: NewPromoteResourceAttributes(config.OTLPConfig{ + PromoteAllResourceAttributes: tc.promoteAllResourceAttributes, + PromoteResourceAttributes: tc.promoteResourceAttributes, + IgnoreResourceAttributes: tc.ignoreResourceAttributes, + }), } lbls := createAttributes(resource, attrs, settings, tc.ignoreAttrs, false, model.MetricNameLabel, "test_metric") diff --git a/storage/remote/otlptranslator/prometheusremotewrite/metrics_to_prw.go b/storage/remote/otlptranslator/prometheusremotewrite/metrics_to_prw.go index 3d0285a185..42717c69fb 100644 --- a/storage/remote/otlptranslator/prometheusremotewrite/metrics_to_prw.go +++ b/storage/remote/otlptranslator/prometheusremotewrite/metrics_to_prw.go @@ -27,10 +27,16 @@ import ( "go.opentelemetry.io/collector/pdata/pmetric" "go.uber.org/multierr" + "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/prompb" "github.com/prometheus/prometheus/util/annotations" ) +type PromoteResourceAttributes struct { + promoteAll bool + attrs map[string]struct{} +} + type Settings struct { Namespace string ExternalLabels map[string]string @@ -38,7 +44,7 @@ type Settings struct { ExportCreatedMetric bool AddMetricSuffixes bool AllowUTF8 bool - PromoteResourceAttributes []string + PromoteResourceAttributes *PromoteResourceAttributes KeepIdentifyingResourceAttributes bool ConvertHistogramsToNHCB bool AllowDeltaTemporality bool @@ -272,3 +278,46 @@ func (c *PrometheusConverter) addSample(sample *prompb.Sample, lbls []prompb.Lab ts.Samples = append(ts.Samples, *sample) return ts } + +func NewPromoteResourceAttributes(otlpCfg config.OTLPConfig) *PromoteResourceAttributes { + attrs := otlpCfg.PromoteResourceAttributes + if otlpCfg.PromoteAllResourceAttributes { + attrs = otlpCfg.IgnoreResourceAttributes + } + attrsMap := make(map[string]struct{}, len(attrs)) + for _, s := range attrs { + attrsMap[s] = struct{}{} + } + return &PromoteResourceAttributes{ + promoteAll: otlpCfg.PromoteAllResourceAttributes, + attrs: attrsMap, + } +} + +// promotedAttributes returns labels for promoted resourceAttributes. +func (s *PromoteResourceAttributes) promotedAttributes(resourceAttributes pcommon.Map) []prompb.Label { + if s == nil { + return nil + } + + var promotedAttrs []prompb.Label + if s.promoteAll { + promotedAttrs = make([]prompb.Label, 0, resourceAttributes.Len()) + resourceAttributes.Range(func(name string, value pcommon.Value) bool { + if _, exists := s.attrs[name]; !exists { + promotedAttrs = append(promotedAttrs, prompb.Label{Name: name, Value: value.AsString()}) + } + return true + }) + } else { + promotedAttrs = make([]prompb.Label, 0, len(s.attrs)) + resourceAttributes.Range(func(name string, value pcommon.Value) bool { + if _, exists := s.attrs[name]; exists { + promotedAttrs = append(promotedAttrs, prompb.Label{Name: name, Value: value.AsString()}) + } + return true + }) + } + sort.Stable(ByLabelName(promotedAttrs)) + return promotedAttrs +} diff --git a/storage/remote/write_handler.go b/storage/remote/write_handler.go index d43edd78bb..21b51ba5ef 100644 --- a/storage/remote/write_handler.go +++ b/storage/remote/write_handler.go @@ -592,7 +592,7 @@ func (rw *rwExporter) ConsumeMetrics(ctx context.Context, md pmetric.Metrics) er annots, err := converter.FromMetrics(ctx, md, otlptranslator.Settings{ AddMetricSuffixes: otlpCfg.TranslationStrategy != config.NoTranslation, AllowUTF8: otlpCfg.TranslationStrategy != config.UnderscoreEscapingWithSuffixes, - PromoteResourceAttributes: otlpCfg.PromoteResourceAttributes, + PromoteResourceAttributes: otlptranslator.NewPromoteResourceAttributes(otlpCfg), KeepIdentifyingResourceAttributes: otlpCfg.KeepIdentifyingResourceAttributes, ConvertHistogramsToNHCB: otlpCfg.ConvertHistogramsToNHCB, AllowDeltaTemporality: rw.allowDeltaTemporality,