diff --git a/promql/promqltest/README.md b/promql/promqltest/README.md
new file mode 100644
index 0000000000..82ba92476e
--- /dev/null
+++ b/promql/promqltest/README.md
@@ -0,0 +1,128 @@
+# The PromQL test scripting language
+
+This package contains two things:
+
+* an implementation of a test scripting language for PromQL engines
+* a predefined set of tests written in that scripting language
+
+The predefined set of tests can be run against any PromQL engine implementation by calling `promqltest.RunBuiltinTests()`.
+Any other test script can be run with `promqltest.RunTest()`.
+
+The rest of this document explains the test scripting language.
+
+Each test script is written in plain text.
+
+Comments can be given by prefixing the comment with a `#`, for example:
+
+```
+# This is a comment.
+```
+
+Each test file contains a series of commands. There are three kinds of commands:
+
+* `load`
+* `clear`
+* `eval`
+
+Each command is executed in the order given in the file.
+
+## `load` command
+
+`load` adds some data to the test environment.
+
+The syntax is as follows:
+
+```
+load <interval>
+    <series> <points>
+    ...
+    <series> <points>
+```
+
+* `<interval>` is the step between points (eg. `1m` or `30s`)
+* `<series>` is a Prometheus series name in the usual `metric{label="value"}` syntax
+* `<points>` is a specification of the points to add for that series, following the same expanding syntax as for `promtool unittest` documented [here](../../docs/configuration/unit_testing_rules.md#series)
+
+For example:
+
+```
+load 1m
+    my_metric{env="prod"} 5 2+3x2 _ stale {{schema:1 sum:3 count:22 buckets:[5 10 7]}}
+```
+
+...will create a single series with labels `my_metric{env="prod"}`, with the following points:
+
+* t=0: value is 5
+* t=1m: value is 2
+* t=2m: value is 5
+* t=3m: value is 7
+* t=4m: no point
+* t=5m: stale marker
+* t=6m: native histogram with schema 1, sum -3, count 22 and bucket counts 5, 10 and 7
+
+Each `load` command is additive - it does not replace any data loaded in a previous `load` command.
+Use `clear` to remove all loaded data.
+
+## `clear` command
+
+`clear` removes all data previously loaded with `load` commands.
+
+## `eval` command
+
+`eval` runs a query against the test environment and asserts that the result is as expected.
+
+Both instant and range queries are supported.
+
+The syntax is as follows:
+
+```
+# Instant query
+eval instant at <time> <query>
+    <series> <points>
+    ...
+    <series> <points>
+    
+# Range query
+eval range from <start> to <end> step <step> <query>
+    <series> <points>
+    ...
+    <series> <points>
+```
+
+* `<time>` is the timestamp to evaluate the instant query at (eg. `1m`)
+* `<start>` and `<end>` specify the time range of the range query, and use the same syntax as `<time>`
+* `<step>` is the step of the range query, and uses the same syntax as `<time>` (eg. `30s`)
+* `<series>` and `<points>` specify the expected values, and follow the same syntax as for `load` above
+
+For example:
+
+```
+eval instant at 1m sum by (env) (my_metric)
+    {env="prod"} 5
+    {env="test"} 20
+    
+eval range from 0 to 3m step 1m sum by (env) (my_metric)
+    {env="prod"} 2 5 10 20
+    {env="test"} 10 20 30 45
+```
+
+Instant queries also support asserting that the series are returned in exactly the order specified: use `eval_ordered instant ...` instead of `eval instant ...`.
+This is not supported for range queries.
+
+It is also possible to test that queries fail: use `eval_fail instant ...` or `eval_fail range ...`.
+`eval_fail` optionally takes an expected error message string or regexp to assert that the error message is as expected.
+
+For example:
+
+```
+# Assert that the query fails for any reason without asserting on the error message.
+eval_fail instant at 1m ceil({__name__=~'testmetric1|testmetric2'})
+
+# Assert that the query fails with exactly the provided error message string.
+eval_fail instant at 1m ceil({__name__=~'testmetric1|testmetric2'})
+    expected_fail_message vector cannot contain metrics with the same labelset
+
+# Assert that the query fails with an error message matching the regexp provided.
+eval_fail instant at 1m ceil({__name__=~'testmetric1|testmetric2'})
+    expected_fail_regexp (vector cannot contain metrics .*|something else went wrong)
+```
diff --git a/promql/promqltest/test.go b/promql/promqltest/test.go
index 1affd91f63..bebb82cf2a 100644
--- a/promql/promqltest/test.go
+++ b/promql/promqltest/test.go
@@ -306,6 +306,21 @@ func (t *test) parseEval(lines []string, i int) (int, *evalCmd, error) {
 			i--
 			break
 		}
+
+		if cmd.fail && strings.HasPrefix(defLine, "expected_fail_message") {
+			cmd.expectedFailMessage = strings.TrimSpace(strings.TrimPrefix(defLine, "expected_fail_message"))
+			break
+		}
+
+		if cmd.fail && strings.HasPrefix(defLine, "expected_fail_regexp") {
+			pattern := strings.TrimSpace(strings.TrimPrefix(defLine, "expected_fail_regexp"))
+			cmd.expectedFailRegexp, err = regexp.Compile(pattern)
+			if err != nil {
+				return i, nil, formatErr("invalid regexp '%s' for expected_fail_regexp: %w", pattern, err)
+			}
+			break
+		}
+
 		if f, err := parseNumber(defLine); err == nil {
 			cmd.expect(0, parser.SequenceValue{Value: f})
 			break
@@ -455,8 +470,10 @@ type evalCmd struct {
 	step  time.Duration
 	line  int
 
-	isRange       bool // if false, instant query
-	fail, ordered bool
+	isRange             bool // if false, instant query
+	fail, ordered       bool
+	expectedFailMessage string
+	expectedFailRegexp  *regexp.Regexp
 
 	metrics  map[uint64]labels.Labels
 	expected map[uint64]entry
@@ -645,6 +662,24 @@ func (ev *evalCmd) compareResult(result parser.Value) error {
 	return nil
 }
 
+func (ev *evalCmd) checkExpectedFailure(actual error) error {
+	if ev.expectedFailMessage != "" {
+		if ev.expectedFailMessage != actual.Error() {
+			return fmt.Errorf("expected error %q evaluating query %q (line %d), but got: %s", ev.expectedFailMessage, ev.expr, ev.line, actual.Error())
+		}
+	}
+
+	if ev.expectedFailRegexp != nil {
+		if !ev.expectedFailRegexp.MatchString(actual.Error()) {
+			return fmt.Errorf("expected error matching pattern %q evaluating query %q (line %d), but got: %s", ev.expectedFailRegexp.String(), ev.expr, ev.line, actual.Error())
+		}
+	}
+
+	// We're not expecting a particular error, or we got the error we expected.
+	// This test passes.
+	return nil
+}
+
 func formatSeriesResult(s promql.Series) string {
 	floatPlural := "s"
 	histogramPlural := "s"
@@ -795,7 +830,7 @@ func (t *test) execRangeEval(cmd *evalCmd, engine promql.QueryEngine) error {
 	res := q.Exec(t.context)
 	if res.Err != nil {
 		if cmd.fail {
-			return nil
+			return cmd.checkExpectedFailure(res.Err)
 		}
 
 		return fmt.Errorf("error evaluating query %q (line %d): %w", cmd.expr, cmd.line, res.Err)
@@ -827,6 +862,10 @@ func (t *test) execInstantEval(cmd *evalCmd, engine promql.QueryEngine) error {
 		res := q.Exec(t.context)
 		if res.Err != nil {
 			if cmd.fail {
+				if err := cmd.checkExpectedFailure(res.Err); err != nil {
+					return err
+				}
+
 				continue
 			}
 			return fmt.Errorf("error evaluating query %q (line %d): %w", iq.expr, cmd.line, res.Err)
diff --git a/promql/promqltest/test_test.go b/promql/promqltest/test_test.go
index f6fe387071..faffb1dd17 100644
--- a/promql/promqltest/test_test.go
+++ b/promql/promqltest/test_test.go
@@ -263,6 +263,60 @@ eval_fail instant at 0m ceil({__name__=~'testmetric1|testmetric2'})
 			input:         `eval_fail instant at 0s vector(0)`,
 			expectedError: `expected error evaluating query "vector(0)" (line 1) but got none`,
 		},
+		"instant query expected to fail with specific error message, and query fails with that error": {
+			input: `
+load 5m
+	testmetric1{src="a",dst="b"} 0
+	testmetric2{src="a",dst="b"} 1
+
+eval_fail instant at 0m ceil({__name__=~'testmetric1|testmetric2'})
+	expected_fail_message vector cannot contain metrics with the same labelset
+`,
+		},
+		"instant query expected to fail with specific error message, and query fails with a different error": {
+			input: `
+load 5m
+	testmetric1{src="a",dst="b"} 0
+	testmetric2{src="a",dst="b"} 1
+
+eval_fail instant at 0m ceil({__name__=~'testmetric1|testmetric2'})
+	expected_fail_message something else went wrong
+`,
+			expectedError: `expected error "something else went wrong" evaluating query "ceil({__name__=~'testmetric1|testmetric2'})" (line 6), but got: vector cannot contain metrics with the same labelset`,
+		},
+
+		"instant query expected to fail with error matching pattern, and query fails with that error": {
+			input: `
+load 5m
+	testmetric1{src="a",dst="b"} 0
+	testmetric2{src="a",dst="b"} 1
+
+eval_fail instant at 0m ceil({__name__=~'testmetric1|testmetric2'})
+	expected_fail_regexp vector .* contain metrics
+`,
+		},
+		"instant query expected to fail with error matching pattern, and query fails with a different error": {
+			input: `
+load 5m
+	testmetric1{src="a",dst="b"} 0
+	testmetric2{src="a",dst="b"} 1
+
+eval_fail instant at 0m ceil({__name__=~'testmetric1|testmetric2'})
+	expected_fail_regexp something else went wrong
+`,
+			expectedError: `expected error matching pattern "something else went wrong" evaluating query "ceil({__name__=~'testmetric1|testmetric2'})" (line 6), but got: vector cannot contain metrics with the same labelset`,
+		},
+		"instant query expected to fail with error matching pattern, and pattern is not a valid regexp": {
+			input: `
+load 5m
+	testmetric1{src="a",dst="b"} 0
+	testmetric2{src="a",dst="b"} 1
+
+eval_fail instant at 0m ceil({__name__=~'testmetric1|testmetric2'})
+	expected_fail_regexp [
+`,
+			expectedError: `error in eval ceil({__name__=~'testmetric1|testmetric2'}) (line 7): invalid regexp '[' for expected_fail_regexp: error parsing regexp: missing closing ]: ` + "`[`",
+		},
 		"instant query with results expected to match provided order, and result is in expected order": {
 			input: testData + `
 eval_ordered instant at 50m sort(http_requests)
@@ -384,6 +438,59 @@ eval_fail range from 0 to 10m step 5m ceil({__name__=~'testmetric1|testmetric2'}
 			input:         `eval_fail range from 0 to 10m step 5m vector(0)`,
 			expectedError: `expected error evaluating query "vector(0)" (line 1) but got none`,
 		},
+		"range query expected to fail with specific error message, and query fails with that error": {
+			input: `
+load 5m
+	testmetric1{src="a",dst="b"} 0
+	testmetric2{src="a",dst="b"} 1
+
+eval_fail range from 0 to 10m step 5m ceil({__name__=~'testmetric1|testmetric2'})
+	expected_fail_message vector cannot contain metrics with the same labelset
+`,
+		},
+		"range query expected to fail with specific error message, and query fails with a different error": {
+			input: `
+load 5m
+	testmetric1{src="a",dst="b"} 0
+	testmetric2{src="a",dst="b"} 1
+
+eval_fail range from 0 to 10m step 5m ceil({__name__=~'testmetric1|testmetric2'})
+	expected_fail_message something else went wrong
+`,
+			expectedError: `expected error "something else went wrong" evaluating query "ceil({__name__=~'testmetric1|testmetric2'})" (line 6), but got: vector cannot contain metrics with the same labelset`,
+		},
+		"range query expected to fail with error matching pattern, and query fails with that error": {
+			input: `
+load 5m
+	testmetric1{src="a",dst="b"} 0
+	testmetric2{src="a",dst="b"} 1
+
+eval_fail range from 0 to 10m step 5m ceil({__name__=~'testmetric1|testmetric2'})
+	expected_fail_regexp vector .* contain metrics
+`,
+		},
+		"range query expected to fail with error matching pattern, and query fails with a different error": {
+			input: `
+load 5m
+	testmetric1{src="a",dst="b"} 0
+	testmetric2{src="a",dst="b"} 1
+
+eval_fail range from 0 to 10m step 5m ceil({__name__=~'testmetric1|testmetric2'})
+	expected_fail_regexp something else went wrong
+`,
+			expectedError: `expected error matching pattern "something else went wrong" evaluating query "ceil({__name__=~'testmetric1|testmetric2'})" (line 6), but got: vector cannot contain metrics with the same labelset`,
+		},
+		"range query expected to fail with error matching pattern, and pattern is not a valid regexp": {
+			input: `
+load 5m
+	testmetric1{src="a",dst="b"} 0
+	testmetric2{src="a",dst="b"} 1
+
+eval_fail range from 0 to 10m step 5m ceil({__name__=~'testmetric1|testmetric2'})
+	expected_fail_regexp [
+`,
+			expectedError: `error in eval ceil({__name__=~'testmetric1|testmetric2'}) (line 7): invalid regexp '[' for expected_fail_regexp: error parsing regexp: missing closing ]: ` + "`[`",
+		},
 		"range query with from and to timestamps in wrong order": {
 			input:         `eval range from 10m to 9m step 5m vector(0)`,
 			expectedError: `error in eval vector(0) (line 1): invalid test definition, end timestamp (9m) is before start timestamp (10m)`,