diff --git a/pkg/textparse/lex.l b/pkg/textparse/lex.l index 4c6260d2de..52f9f46e44 100644 --- a/pkg/textparse/lex.l +++ b/pkg/textparse/lex.l @@ -27,6 +27,8 @@ func (l *lexer) Lex() int { return eof } c := l.b[l.i] + + l.offsets = l.offsets[:0] %} D [0-9] @@ -47,19 +49,25 @@ M [a-zA-Z_:] #[^\r\n]*\n l.mstart = l.i [\r\n \t]+ l.mstart = l.i -{L}({L}|{D})*\{ s = lstateLName +{L}({L}|{D})*\{ s = lstateLabels + l.offsets = append(l.offsets, l.i-1) {L}({L}|{D})* s = lstateValue l.mend = l.i + l.offsets = append(l.offsets, l.i) [ \t]+ \} s = lstateValue l.mend = l.i ,? s = lstateLName + l.offsets = append(l.offsets, l.i) -{M}({M}|{D})*= s = lstateLValue +{M}({M}|{D})*= s = lstateLValue + l.offsets = append(l.offsets, l.i-1) \"(\\.|[^\\"])*\" s = lstateLabels + l.offsets = append(l.offsets, l.i-1) \'(\\.|[^\\'])*\' s = lstateLabels + l.offsets = append(l.offsets, l.i-1) [ \t]+ l.vstart = l.i (NaN) l.val = math.NaN() diff --git a/pkg/textparse/lex.l.go b/pkg/textparse/lex.l.go index fc76fe8d0c..6fdb7dd9a2 100644 --- a/pkg/textparse/lex.l.go +++ b/pkg/textparse/lex.l.go @@ -28,6 +28,8 @@ func (l *lexer) Lex() int { } c := l.b[l.i] + l.offsets = l.offsets[:0] + yystate0: switch yyt := s; yyt { @@ -309,13 +311,15 @@ yyrule3: // [\r\n \t]+ } yyrule4: // {L}({L}|{D})*\{ { - s = lstateLName + s = lstateLabels + l.offsets = append(l.offsets, l.i-1) goto yystate0 } yyrule5: // {L}({L}|{D})* { s = lstateValue l.mend = l.i + l.offsets = append(l.offsets, l.i) goto yystate0 } yyrule6: // [ \t]+ @@ -330,21 +334,25 @@ yyrule7: // \} yyrule8: // ,? { s = lstateLName + l.offsets = append(l.offsets, l.i) goto yystate0 } yyrule9: // {M}({M}|{D})*= { s = lstateLValue + l.offsets = append(l.offsets, l.i-1) goto yystate0 } yyrule10: // \"(\\.|[^\\"])*\" { s = lstateLabels + l.offsets = append(l.offsets, l.i-1) goto yystate0 } yyrule11: // \'(\\.|[^\\'])*\' { s = lstateLabels + l.offsets = append(l.offsets, l.i-1) goto yystate0 } yyrule12: // [ \t]+ diff --git a/pkg/textparse/parse.go b/pkg/textparse/parse.go index bf7d93d8df..8139120c62 100644 --- a/pkg/textparse/parse.go +++ b/pkg/textparse/parse.go @@ -5,9 +5,10 @@ import ( "errors" "io" "reflect" + "sort" "unsafe" - "k8s.io/client-go/pkg/labels" + "github.com/prometheus/prometheus/pkg/labels" ) type lexer struct { @@ -15,9 +16,10 @@ type lexer struct { i int vstart int - mstart, mend int err error val float64 + offsets []int + mstart, mend int } const eof = 0 @@ -70,8 +72,22 @@ func (p *Parser) Err() error { return p.l.err } -func (p *Parser) Metric() labels.Labels { - return nil +func (p *Parser) Metric(l *labels.Labels) { + *l = append(*l, labels.Label{ + Name: labels.MetricName, + Value: string(p.l.b[p.l.mstart:p.l.offsets[0]]), + }) + + for i := 1; i < len(p.l.offsets); i += 3 { + a, b, c := p.l.offsets[i], p.l.offsets[i+1], p.l.offsets[i+2] + + *l = append(*l, labels.Label{ + Name: string(p.l.b[a:b]), + Value: string(p.l.b[b+2 : c]), + }) + } + + sort.Sort((*l)[1:]) } func yoloString(b []byte) string { diff --git a/pkg/textparse/parse_test.go b/pkg/textparse/parse_test.go index b5e513fafe..3607990226 100644 --- a/pkg/textparse/parse_test.go +++ b/pkg/textparse/parse_test.go @@ -10,9 +10,69 @@ import ( "github.com/stretchr/testify/require" ) +func TestParse(t *testing.T) { + input := `# HELP go_gc_duration_seconds A summary of the GC invocation durations. +# TYPE go_gc_duration_seconds summary +go_gc_duration_seconds{quantile="0"} 4.9351e-05 +go_gc_duration_seconds{quantile="0.25"} 7.424100000000001e-05 +go_gc_duration_seconds{quantile="0.5",a="b"} 8.3835e-05 +go_gc_duration_seconds_count 99 +# HELP go_goroutines Number of goroutines that currently exist. +# TYPE go_goroutines gauge +go_goroutines 33` + + exp := []struct { + lset labels.Labels + m string + v float64 + }{ + { + m: `go_gc_duration_seconds{quantile="0"}`, + v: 4.9351e-05, + lset: labels.FromStrings("__name__", "go_gc_duration_seconds", "quantile", "0"), + }, { + m: `go_gc_duration_seconds{quantile="0.25"}`, + v: 7.424100000000001e-05, + lset: labels.FromStrings("__name__", "go_gc_duration_seconds", "quantile", "0.25"), + }, { + m: `go_gc_duration_seconds{quantile="0.5",a="b"}`, + v: 8.3835e-05, + lset: labels.FromStrings("__name__", "go_gc_duration_seconds", "quantile", "0.5", "a", "b"), + }, { + m: `go_gc_duration_seconds_count`, + v: 99, + lset: labels.FromStrings("__name__", "go_gc_duration_seconds_count"), + }, { + m: `go_goroutines`, + v: 33, + lset: labels.FromStrings("__name__", "go_goroutines"), + }, + } + + p := New([]byte(input)) + i := 0 + + var res labels.Labels + + for p.Next() { + m, _, v := p.At() + + p.Metric(&res) + + require.Equal(t, exp[i].m, string(m)) + require.Equal(t, exp[i].v, v) + require.Equal(t, exp[i].lset, res) + + i++ + res = res[:0] + } + + require.NoError(t, p.Err()) + +} + func BenchmarkParse(b *testing.B) { f, err := os.Open("testdata.txt") - // f, err := os.Open("../../../../fabxc/tsdb/cmd/tsdb/testdata.1m") require.NoError(b, err) defer f.Close() @@ -21,7 +81,7 @@ func BenchmarkParse(b *testing.B) { buf, err := ioutil.ReadAll(f) require.NoError(b, err) - for i := 0; i*1e6 < b.N; i++ { + for i := 0; i*527 < b.N; i++ { tbuf = append(tbuf, buf...) tbuf = append(tbuf, '\n') } @@ -30,7 +90,6 @@ func BenchmarkParse(b *testing.B) { i := 0 var m labels.Labels total := 0 - // res := make(labels.Labels, 0, 5) b.ResetTimer() b.ReportAllocs() @@ -39,13 +98,12 @@ func BenchmarkParse(b *testing.B) { for p.Next() && i < b.N { p.At() - // res = res[:0] + res := make(labels.Labels, 0, 5) + p.Metric(&res) - // err = ParseMetric(mb, &res) - // require.NoError(b, err) - - // total += len(res) + total += len(res) i++ + res = res[:0] } _ = m fmt.Println(total)