From eff3a13e196bee80d0247020fca293da3e990476 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Sat, 25 Mar 2023 13:31:24 +0000 Subject: [PATCH] model/textparse: parsers take a labels SymbolTable This allows strings to be interned to save memory. Signed-off-by: Bryan Boreham --- cmd/promtool/backfill.go | 5 +++-- model/textparse/interface.go | 12 ++++++------ model/textparse/interface_test.go | 4 +++- model/textparse/openmetricsparse.go | 7 +++++-- model/textparse/openmetricsparse_test.go | 8 ++++---- model/textparse/promparse.go | 7 +++++-- model/textparse/promparse_test.go | 19 +++++++++++-------- model/textparse/protobufparse.go | 3 ++- model/textparse/protobufparse_test.go | 4 ++-- web/federate_test.go | 2 +- 10 files changed, 42 insertions(+), 29 deletions(-) diff --git a/cmd/promtool/backfill.go b/cmd/promtool/backfill.go index 39410881b2..601c3ced9f 100644 --- a/cmd/promtool/backfill.go +++ b/cmd/promtool/backfill.go @@ -127,7 +127,8 @@ func createBlocks(input []byte, mint, maxt, maxBlockDuration int64, maxSamplesIn ctx := context.Background() app := w.Appender(ctx) - p := textparse.NewOpenMetricsParser(input) + symbolTable := labels.NewSymbolTable() // One table per block means it won't grow too large. + p := textparse.NewOpenMetricsParser(input, symbolTable) samplesCount := 0 for { e, err := p.Next() @@ -216,7 +217,7 @@ func createBlocks(input []byte, mint, maxt, maxBlockDuration int64, maxSamplesIn } func backfill(maxSamplesInAppender int, input []byte, outputDir string, humanReadable, quiet bool, maxBlockDuration time.Duration) (err error) { - p := textparse.NewOpenMetricsParser(input) + p := textparse.NewOpenMetricsParser(input, nil) // Don't need a SymbolTable to get max and min timestamps. maxt, mint, err := getMinAndMaxTimestamps(p) if err != nil { return fmt.Errorf("getting min and max timestamp: %w", err) diff --git a/model/textparse/interface.go b/model/textparse/interface.go index 3a363ebfbc..2e8c40e72f 100644 --- a/model/textparse/interface.go +++ b/model/textparse/interface.go @@ -80,22 +80,22 @@ type Parser interface { // // This function always returns a valid parser, but might additionally // return an error if the content type cannot be parsed. -func New(b []byte, contentType string, parseClassicHistograms bool) (Parser, error) { +func New(b []byte, contentType string, parseClassicHistograms bool, st *labels.SymbolTable) (Parser, error) { if contentType == "" { - return NewPromParser(b), nil + return NewPromParser(b, st), nil } mediaType, _, err := mime.ParseMediaType(contentType) if err != nil { - return NewPromParser(b), err + return NewPromParser(b, st), err } switch mediaType { case "application/openmetrics-text": - return NewOpenMetricsParser(b), nil + return NewOpenMetricsParser(b, st), nil case "application/vnd.google.protobuf": - return NewProtobufParser(b, parseClassicHistograms), nil + return NewProtobufParser(b, parseClassicHistograms, st), nil default: - return NewPromParser(b), nil + return NewPromParser(b, st), nil } } diff --git a/model/textparse/interface_test.go b/model/textparse/interface_test.go index de140d6819..c644565628 100644 --- a/model/textparse/interface_test.go +++ b/model/textparse/interface_test.go @@ -17,6 +17,8 @@ import ( "testing" "github.com/stretchr/testify/require" + + "github.com/prometheus/prometheus/model/labels" ) func TestNewParser(t *testing.T) { @@ -91,7 +93,7 @@ func TestNewParser(t *testing.T) { tt := tt // Copy to local variable before going parallel. t.Parallel() - p, err := New([]byte{}, tt.contentType, false) + p, err := New([]byte{}, tt.contentType, false, labels.NewSymbolTable()) tt.validateParser(t, p) if tt.err == "" { require.NoError(t, err) diff --git a/model/textparse/openmetricsparse.go b/model/textparse/openmetricsparse.go index 2a7eae080f..ea92bc55f4 100644 --- a/model/textparse/openmetricsparse.go +++ b/model/textparse/openmetricsparse.go @@ -97,8 +97,11 @@ type OpenMetricsParser struct { } // NewOpenMetricsParser returns a new parser of the byte slice. -func NewOpenMetricsParser(b []byte) Parser { - return &OpenMetricsParser{l: &openMetricsLexer{b: b}} +func NewOpenMetricsParser(b []byte, st *labels.SymbolTable) Parser { + return &OpenMetricsParser{ + l: &openMetricsLexer{b: b}, + builder: labels.NewScratchBuilderWithSymbolTable(st, 16), + } } // Series returns the bytes of the series, the timestamp if set, and the value diff --git a/model/textparse/openmetricsparse_test.go b/model/textparse/openmetricsparse_test.go index f3aa21dfa9..e356beeae5 100644 --- a/model/textparse/openmetricsparse_test.go +++ b/model/textparse/openmetricsparse_test.go @@ -247,7 +247,7 @@ foo_total 17.0 1520879607.789 # {id="counter-test"} 5` }, } - p := NewOpenMetricsParser([]byte(input)) + p := NewOpenMetricsParser([]byte(input), labels.NewSymbolTable()) i := 0 var res labels.Labels @@ -378,7 +378,7 @@ choices}`, "strange©™\n'quoted' \"name\"", "6"), }, } - p := NewOpenMetricsParser([]byte(input)) + p := NewOpenMetricsParser([]byte(input), labels.NewSymbolTable()) i := 0 var res labels.Labels @@ -727,7 +727,7 @@ func TestOpenMetricsParseErrors(t *testing.T) { } for i, c := range cases { - p := NewOpenMetricsParser([]byte(c.input)) + p := NewOpenMetricsParser([]byte(c.input), labels.NewSymbolTable()) var err error for err == nil { _, err = p.Next() @@ -792,7 +792,7 @@ func TestOMNullByteHandling(t *testing.T) { } for i, c := range cases { - p := NewOpenMetricsParser([]byte(c.input)) + p := NewOpenMetricsParser([]byte(c.input), labels.NewSymbolTable()) var err error for err == nil { _, err = p.Next() diff --git a/model/textparse/promparse.go b/model/textparse/promparse.go index 1de783b0d0..54b5306e6a 100644 --- a/model/textparse/promparse.go +++ b/model/textparse/promparse.go @@ -166,8 +166,11 @@ type PromParser struct { } // NewPromParser returns a new parser of the byte slice. -func NewPromParser(b []byte) Parser { - return &PromParser{l: &promlexer{b: append(b, '\n')}} +func NewPromParser(b []byte, st *labels.SymbolTable) Parser { + return &PromParser{ + l: &promlexer{b: append(b, '\n')}, + builder: labels.NewScratchBuilderWithSymbolTable(st, 16), + } } // Series returns the bytes of the series, the timestamp if set, and the value diff --git a/model/textparse/promparse_test.go b/model/textparse/promparse_test.go index 775e5faa56..4ec8c9b9cd 100644 --- a/model/textparse/promparse_test.go +++ b/model/textparse/promparse_test.go @@ -178,7 +178,7 @@ testmetric{label="\"bar\""} 1` }, } - p := NewPromParser([]byte(input)) + p := NewPromParser([]byte(input), labels.NewSymbolTable()) i := 0 var res labels.Labels @@ -304,7 +304,7 @@ choices}`, "strange©™\n'quoted' \"name\"", "6"), }, } - p := NewPromParser([]byte(input)) + p := NewPromParser([]byte(input), labels.NewSymbolTable()) i := 0 var res labels.Labels @@ -422,7 +422,7 @@ func TestPromParseErrors(t *testing.T) { } for i, c := range cases { - p := NewPromParser([]byte(c.input)) + p := NewPromParser([]byte(c.input), labels.NewSymbolTable()) var err error for err == nil { _, err = p.Next() @@ -476,7 +476,7 @@ func TestPromNullByteHandling(t *testing.T) { } for i, c := range cases { - p := NewPromParser([]byte(c.input)) + p := NewPromParser([]byte(c.input), labels.NewSymbolTable()) var err error for err == nil { _, err = p.Next() @@ -497,7 +497,7 @@ const ( ) func BenchmarkParse(b *testing.B) { - for parserName, parser := range map[string]func([]byte) Parser{ + for parserName, parser := range map[string]func([]byte, *labels.SymbolTable) Parser{ "prometheus": NewPromParser, "openmetrics": NewOpenMetricsParser, } { @@ -516,8 +516,9 @@ func BenchmarkParse(b *testing.B) { b.ReportAllocs() b.ResetTimer() + st := labels.NewSymbolTable() for i := 0; i < b.N; i += promtestdataSampleCount { - p := parser(buf) + p := parser(buf, st) Outer: for i < b.N { @@ -544,8 +545,9 @@ func BenchmarkParse(b *testing.B) { b.ReportAllocs() b.ResetTimer() + st := labels.NewSymbolTable() for i := 0; i < b.N; i += promtestdataSampleCount { - p := parser(buf) + p := parser(buf, st) Outer: for i < b.N { @@ -577,8 +579,9 @@ func BenchmarkParse(b *testing.B) { b.ReportAllocs() b.ResetTimer() + st := labels.NewSymbolTable() for i := 0; i < b.N; i += promtestdataSampleCount { - p := parser(buf) + p := parser(buf, st) Outer: for i < b.N { diff --git a/model/textparse/protobufparse.go b/model/textparse/protobufparse.go index 8fd89af825..ea3a2e1a34 100644 --- a/model/textparse/protobufparse.go +++ b/model/textparse/protobufparse.go @@ -80,13 +80,14 @@ type ProtobufParser struct { } // NewProtobufParser returns a parser for the payload in the byte slice. -func NewProtobufParser(b []byte, parseClassicHistograms bool) Parser { +func NewProtobufParser(b []byte, parseClassicHistograms bool, st *labels.SymbolTable) Parser { return &ProtobufParser{ in: b, state: EntryInvalid, mf: &dto.MetricFamily{}, metricBytes: &bytes.Buffer{}, parseClassicHistograms: parseClassicHistograms, + builder: labels.NewScratchBuilderWithSymbolTable(st, 16), } } diff --git a/model/textparse/protobufparse_test.go b/model/textparse/protobufparse_test.go index c807ae644c..e323a6cc8f 100644 --- a/model/textparse/protobufparse_test.go +++ b/model/textparse/protobufparse_test.go @@ -743,7 +743,7 @@ func TestProtobufParse(t *testing.T) { }{ { name: "ignore classic buckets of native histograms", - parser: NewProtobufParser(inputBuf.Bytes(), false), + parser: NewProtobufParser(inputBuf.Bytes(), false, labels.NewSymbolTable()), expected: []parseResult{ { m: "go_build_info", @@ -1280,7 +1280,7 @@ func TestProtobufParse(t *testing.T) { }, { name: "parse classic and native buckets", - parser: NewProtobufParser(inputBuf.Bytes(), true), + parser: NewProtobufParser(inputBuf.Bytes(), true, labels.NewSymbolTable()), expected: []parseResult{ { // 0 m: "go_build_info", diff --git a/web/federate_test.go b/web/federate_test.go index 92b806fe89..b8749dfa32 100644 --- a/web/federate_test.go +++ b/web/federate_test.go @@ -391,7 +391,7 @@ func TestFederationWithNativeHistograms(t *testing.T) { body, err := io.ReadAll(res.Body) require.NoError(t, err) - p := textparse.NewProtobufParser(body, false) + p := textparse.NewProtobufParser(body, false, labels.NewSymbolTable()) var actVec promql.Vector metricFamilies := 0 l := labels.Labels{}