From 63219647385ce111c36453d620edb18c37ba05f5 Mon Sep 17 00:00:00 2001 From: Fabian Reinartz Date: Mon, 11 May 2015 15:56:35 +0200 Subject: [PATCH] Add parsing and execution of new test format. This commit adds a new test structure that parses and executes the new testing language. --- promql/engine.go | 11 +- promql/parse_test.go | 2 +- promql/promql_test.go | 10 +- promql/setup_test.go | 3 +- promql/test.go | 507 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 523 insertions(+), 10 deletions(-) create mode 100644 promql/test.go diff --git a/promql/engine.go b/promql/engine.go index 7b53f661d8..07799ba321 100644 --- a/promql/engine.go +++ b/promql/engine.go @@ -279,20 +279,25 @@ func (ng *Engine) NewRangeQuery(qs string, start, end clientmodel.Timestamp, int if err != nil { return nil, err } + qry := ng.newQuery(expr, start, end, interval) + qry.q = qs + + return qry, nil +} + +func (ng *Engine) newQuery(expr Expr, start, end clientmodel.Timestamp, interval time.Duration) *query { es := &EvalStmt{ Expr: expr, Start: start, End: end, Interval: interval, } - qry := &query{ - q: qs, stmts: Statements{es}, ng: ng, stats: stats.NewTimerGroup(), } - return qry, nil + return qry } // testStmt is an internal helper statement that allows execution diff --git a/promql/parse_test.go b/promql/parse_test.go index ca2ef45555..9940f04684 100644 --- a/promql/parse_test.go +++ b/promql/parse_test.go @@ -1272,7 +1272,7 @@ var testSeries = []struct { }, } -// For these tests only, we use the samallest float64 to signal an omitted value. +// For these tests only, we use the smallest float64 to signal an omitted value. const none = math.SmallestNonzeroFloat64 func newSeq(vals ...float64) (res []sequenceValue) { diff --git a/promql/promql_test.go b/promql/promql_test.go index 93af620d3e..4201daceae 100644 --- a/promql/promql_test.go +++ b/promql/promql_test.go @@ -33,13 +33,13 @@ var ( testEvalTime = testStartTime.Add(testSampleInterval * 10) fixturesPath = "fixtures" - reSample = regexp.MustCompile(`^(.*)(?: \=\>|:) (\-?\d+\.?\d*(?:e-?\d+)?|[+-]Inf|NaN) \@\[(\d+)\]$`) - minNormal = math.Float64frombits(0x0010000000000000) // The smallest positive normal value of type float64. + reSample = regexp.MustCompile(`^(.*)(?: \=\>|:) (\-?\d+\.?\d*(?:e-?\d+)?|[+-]Inf|NaN) \@\[(\d+)\]$`) + // minNormal = math.Float64frombits(0x0010000000000000) // The smallest positive normal value of type float64. ) -const ( - epsilon = 0.000001 // Relative error allowed for sample values. -) +// const ( +// epsilon = 0.000001 // Relative error allowed for sample values. +// ) func annotateWithTime(lines []string, timestamp clientmodel.Timestamp) []string { annotatedLines := []string{} diff --git a/promql/setup_test.go b/promql/setup_test.go index 5796d7f874..796c93896c 100644 --- a/promql/setup_test.go +++ b/promql/setup_test.go @@ -23,7 +23,8 @@ import ( ) var testSampleInterval = time.Duration(5) * time.Minute -var testStartTime = clientmodel.Timestamp(0) + +// var testStartTime = clientmodel.Timestamp(0) func getTestValueStream(startVal, endVal, stepVal clientmodel.SampleValue, startTime clientmodel.Timestamp) (resultValues metric.Values) { currentTime := startTime diff --git a/promql/test.go b/promql/test.go new file mode 100644 index 0000000000..1d3bf66070 --- /dev/null +++ b/promql/test.go @@ -0,0 +1,507 @@ +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package promql + +import ( + "fmt" + "io/ioutil" + "math" + "regexp" + "strconv" + "strings" + "testing" + "time" + + clientmodel "github.com/prometheus/client_golang/model" + + "github.com/prometheus/prometheus/storage" + "github.com/prometheus/prometheus/storage/local" + "github.com/prometheus/prometheus/storage/metric" + "github.com/prometheus/prometheus/utility" + + testutil "github.com/prometheus/prometheus/utility/test" +) + +var ( + minNormal = math.Float64frombits(0x0010000000000000) // The smallest positive normal value of type float64. + + patSpace = regexp.MustCompile("[\t ]+") + patLoad = regexp.MustCompile(`^load\s+(.+?)$`) + patEvalInstant = regexp.MustCompile(`^eval(?:_(fail|ordered))?\s+instant\s+(?:at\s+(.+?))?\s+(.+)$`) +) + +const ( + testStartTime = clientmodel.Timestamp(0) + epsilon = 0.000001 // Relative error allowed for sample values. + maxErrorCount = 10 +) + +// Test is a sequence of read and write commands that are run +// against a test storage. +type Test struct { + *testing.T + + cmds []testCommand + + storage local.Storage + closeStorage func() + queryEngine *Engine +} + +// NewTest returns an initialized empty Test. +func NewTest(t *testing.T, input string) (*Test, error) { + test := &Test{ + T: t, + cmds: []testCommand{}, + } + err := test.parse(input) + test.clear() + + return test, err +} + +func NewTestFromFile(t *testing.T, filename string) (*Test, error) { + content, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + return NewTest(t, string(content)) +} + +func raise(line int, format string, v ...interface{}) error { + return &ParseErr{ + Line: line + 1, + Err: fmt.Errorf(format, v...), + } +} + +func (t *Test) parseLoad(lines []string, i int) (int, *loadCmd, error) { + if !patLoad.MatchString(lines[i]) { + return i, nil, raise(i, "invalid load command. (load )") + } + parts := patLoad.FindStringSubmatch(lines[i]) + + gap, err := utility.StringToDuration(parts[1]) + if err != nil { + return i, nil, raise(i, "invalid step definition %q: %s", parts[1], err) + } + cmd := newLoadCmd(gap) + for i+1 < len(lines) { + i++ + defLine := lines[i] + if len(defLine) == 0 { + i-- + break + } + metric, vals, err := parseSeriesDesc(defLine) + if err != nil { + perr := err.(*ParseErr) + perr.Line = i + 1 + return i, nil, err + } + cmd.set(metric, vals...) + } + return i, cmd, nil +} + +func (t *Test) parseEval(lines []string, i int) (int, *evalCmd, error) { + if !patEvalInstant.MatchString(lines[i]) { + return i, nil, raise(i, "invalid evaluation command. (eval[_fail|_ordered] instant [at ] ") + } + parts := patEvalInstant.FindStringSubmatch(lines[i]) + var ( + mod = parts[1] + at = parts[2] + qry = parts[3] + ) + expr, err := ParseExpr(qry) + if err != nil { + perr := err.(*ParseErr) + perr.Line = i + 1 + perr.Pos += strings.Index(lines[i], qry) + return i, nil, perr + } + + offset, err := utility.StringToDuration(at) + if err != nil { + return i, nil, raise(i, "invalid step definition %q: %s", parts[1], err) + } + ts := testStartTime.Add(offset) + + cmd := newEvalCmd(expr, ts, ts, 0) + switch mod { + case "ordered": + cmd.ordered = true + case "fail": + cmd.fail = true + } + + for j := 1; i+1 < len(lines); j++ { + i++ + defLine := lines[i] + if len(defLine) == 0 { + i-- + break + } + if f, err := parseNumber(defLine); err == nil { + cmd.expect(0, nil, sequenceValue{value: clientmodel.SampleValue(f)}) + break + } + metric, vals, err := parseSeriesDesc(defLine) + if err != nil { + perr := err.(*ParseErr) + perr.Line = i + 1 + return i, nil, err + } + + // Currently, we are not expecting any matrices. + if len(vals) > 1 { + return i, nil, raise(i, "expecting multiple values in instant evaluation not allowed") + } + cmd.expect(j, metric, vals...) + } + return i, cmd, nil +} + +// parse the given command sequence and appends it to the test. +func (t *Test) parse(input string) error { + // Trim lines and remove comments. + lines := strings.Split(input, "\n") + for i, l := range lines { + l = strings.TrimSpace(l) + if strings.HasPrefix(l, "#") { + l = "" + } + lines[i] = l + } + var err error + + // Scan for steps line by line. + for i := 0; i < len(lines); i++ { + l := lines[i] + if len(l) == 0 { + continue + } + var cmd testCommand + + switch c := strings.ToLower(patSpace.Split(l, 2)[0]); { + case c == "clear": + cmd = &clearCmd{} + case c == "load": + i, cmd, err = t.parseLoad(lines, i) + case strings.HasPrefix(c, "eval"): + i, cmd, err = t.parseEval(lines, i) + default: + return raise(i, "invalid command %q", l) + } + if err != nil { + return err + } + t.cmds = append(t.cmds, cmd) + } + return nil +} + +// testCommand is an interface that ensures that only the package internal +// types can be a valid command for a test. +type testCommand interface { + testCmd() +} + +func (*clearCmd) testCmd() {} +func (*loadCmd) testCmd() {} +func (*evalCmd) testCmd() {} + +// loadCmd is a command that loads sequences of sample values for specific +// metrics into the storage. +type loadCmd struct { + gap time.Duration + metrics map[clientmodel.Fingerprint]clientmodel.Metric + defs map[clientmodel.Fingerprint]metric.Values +} + +func newLoadCmd(gap time.Duration) *loadCmd { + return &loadCmd{ + gap: gap, + metrics: map[clientmodel.Fingerprint]clientmodel.Metric{}, + defs: map[clientmodel.Fingerprint]metric.Values{}, + } +} + +func (cmd loadCmd) String() string { + return "load" +} + +// set a sequence of sample values for the given metric. +func (cmd *loadCmd) set(m clientmodel.Metric, vals ...sequenceValue) { + fp := m.Fingerprint() + + samples := make(metric.Values, 0, len(vals)) + ts := testStartTime + for _, v := range vals { + if !v.omitted { + samples = append(samples, metric.SamplePair{ + Timestamp: ts, + Value: v.value, + }) + } + ts = ts.Add(cmd.gap) + } + cmd.defs[fp] = samples + cmd.metrics[fp] = m +} + +// append the defined time series to the storage. +func (cmd *loadCmd) append(a storage.SampleAppender) { + for fp, samples := range cmd.defs { + met := cmd.metrics[fp] + for _, smpl := range samples { + s := &clientmodel.Sample{ + Metric: met, + Value: smpl.Value, + Timestamp: smpl.Timestamp, + } + a.Append(s) + } + } +} + +// evalCmd is a command that evaluates an expression for the given time (range) +// and expects a specific result. +type evalCmd struct { + expr Expr + start, end clientmodel.Timestamp + interval time.Duration + + instant bool + fail, ordered bool + + metrics map[clientmodel.Fingerprint]clientmodel.Metric + expected map[clientmodel.Fingerprint]entry +} + +type entry struct { + pos int + vals []sequenceValue +} + +func (e entry) String() string { + return fmt.Sprintf("%d: %s", e.pos, e.vals) +} + +func newEvalCmd(expr Expr, start, end clientmodel.Timestamp, interval time.Duration) *evalCmd { + return &evalCmd{ + expr: expr, + start: start, + end: end, + interval: interval, + instant: start == end && interval == 0, + + metrics: map[clientmodel.Fingerprint]clientmodel.Metric{}, + expected: map[clientmodel.Fingerprint]entry{}, + } +} + +func (ev *evalCmd) String() string { + return "eval" +} + +// expect adds a new metric with a sequence of values to the set of expected +// results for the query. +func (ev *evalCmd) expect(pos int, m clientmodel.Metric, vals ...sequenceValue) { + if m == nil { + ev.expected[0] = entry{pos: pos, vals: vals} + return + } + fp := m.Fingerprint() + ev.metrics[fp] = m + ev.expected[fp] = entry{pos: pos, vals: vals} +} + +// compareResult compares the result value with the defined expectation. +func (ev *evalCmd) compareResult(result Value) error { + switch val := result.(type) { + case Matrix: + if ev.instant { + return fmt.Errorf("received range result on instant evaluation") + } + seen := map[clientmodel.Fingerprint]bool{} + for pos, v := range val { + fp := v.Metric.Metric.Fingerprint() + if _, ok := ev.metrics[fp]; !ok { + return fmt.Errorf("unexpected metric %s in result", v.Metric.Metric) + } + exp := ev.expected[fp] + if ev.ordered && exp.pos != pos+1 { + return fmt.Errorf("expected metric %s with %v at position %d but was at %d", v.Metric.Metric, exp.vals, exp.pos, pos+1) + } + for i, expVal := range exp.vals { + if !almostEqual(float64(expVal.value), float64(v.Values[i].Value)) { + return fmt.Errorf("expected %v for %s but got %v", expVal, v.Metric.Metric, v.Values) + } + } + seen[fp] = true + } + for fp, expVals := range ev.expected { + if !seen[fp] { + return fmt.Errorf("expected metric %s with %v not found", ev.metrics[fp], expVals) + } + } + + case Vector: + if !ev.instant { + fmt.Errorf("received instant result on range evaluation") + } + seen := map[clientmodel.Fingerprint]bool{} + for pos, v := range val { + fp := v.Metric.Metric.Fingerprint() + if _, ok := ev.metrics[fp]; !ok { + return fmt.Errorf("unexpected metric %s in result", v.Metric.Metric) + } + exp := ev.expected[fp] + if ev.ordered && exp.pos != pos+1 { + return fmt.Errorf("expected metric %s with %v at position %d but was at %d", v.Metric.Metric, exp.vals, exp.pos, pos+1) + } + if !almostEqual(float64(exp.vals[0].value), float64(v.Value)) { + return fmt.Errorf("expected %v for %s but got %v", exp.vals[0].value, v.Metric.Metric, v.Value) + } + + seen[fp] = true + } + for fp, expVals := range ev.expected { + if !seen[fp] { + return fmt.Errorf("expected metric %s with %v not found", ev.metrics[fp], expVals) + } + } + + case *Scalar: + if !almostEqual(float64(ev.expected[0].vals[0].value), float64(val.Value)) { + return fmt.Errorf("expected scalar %v but got %v", val.Value, ev.expected[0].vals[0].value) + } + + default: + panic(fmt.Errorf("promql.Test.compareResult: unexpected result type %T", result)) + } + return nil +} + +// clearCmd is a command that wipes the test's storage state. +type clearCmd struct{} + +func (cmd clearCmd) String() string { + return "clear" +} + +// Run executes the command sequence of the test. Until the maximum error number +// is reached, evaluation errors do not terminate execution. +func (t *Test) Run() error { + for _, cmd := range t.cmds { + err := t.exec(cmd) + // TODO(fabxc): aggregate command errors, yield diffs for result + // comparison errors. + if err != nil { + return err + } + } + return nil +} + +// exec processes a single step of the test +func (t *Test) exec(tc testCommand) error { + switch cmd := tc.(type) { + case *clearCmd: + t.clear() + + case *loadCmd: + cmd.append(t.storage) + t.storage.WaitForIndexing() + + case *evalCmd: + q := t.queryEngine.newQuery(cmd.expr, cmd.start, cmd.end, cmd.interval) + res := q.Exec() + if res.Err != nil { + if cmd.fail { + return nil + } + return fmt.Errorf("error evaluating query: %s", res.Err) + } + if res.Err == nil && cmd.fail { + return fmt.Errorf("expected error evaluating query but got none") + } + + err := cmd.compareResult(res.Value) + if err != nil { + return fmt.Errorf("error in %s %s: %s", cmd, cmd.expr, err) + } + + default: + panic("promql.Test.exec: unknown test command type") + } + return nil +} + +// clear the current test storage of all inserted samples. +func (t *Test) clear() { + if t.closeStorage != nil { + t.closeStorage() + } + if t.queryEngine != nil { + t.queryEngine.Stop() + } + + var closer testutil.Closer + t.storage, closer = local.NewTestStorage(t, 1) + + t.closeStorage = closer.Close + t.queryEngine = NewEngine(t.storage) +} + +func (t *Test) Close() { + t.queryEngine.Stop() + t.closeStorage() +} + +// samplesAlmostEqual returns true if the two sample lines only differ by a +// small relative error in their sample value. +func almostEqual(a, b float64) bool { + // NaN has no equality but for testing we still want to know whether both values + // are NaN. + if math.IsNaN(a) && math.IsNaN(b) { + return true + } + + // Cf. http://floating-point-gui.de/errors/comparison/ + if a == b { + return true + } + + diff := math.Abs(a - b) + + if a == 0 || b == 0 || diff < minNormal { + return diff < epsilon*minNormal + } + return diff/(math.Abs(a)+math.Abs(b)) < epsilon +} + +func parseNumber(s string) (float64, error) { + n, err := strconv.ParseInt(s, 0, 64) + f := float64(n) + if err != nil { + f, err = strconv.ParseFloat(s, 64) + } + if err != nil { + return 0, fmt.Errorf("error parsing number: %s", err) + } + return f, nil +}