diff --git a/promql/engine.go b/promql/engine.go
index 8f922abaab..6ba6008b19 100644
--- a/promql/engine.go
+++ b/promql/engine.go
@@ -2936,17 +2936,15 @@ func (ev *evaluator) VectorBinop(op parser.ItemType, lhs, rhs Vector, matching *
 		if info != nil {
 			lastErr = info
 		}
-		switch {
-		case returnBool:
+		if returnBool {
 			histogramValue = nil
 			if keep {
 				floatValue = 1.0
 			} else {
 				floatValue = 0.0
 			}
-		case !keep:
-			continue
 		}
+
 		metric := resultMetric(ls.Metric, rs.Metric, op, matching, enh)
 		if !ev.enableDelayedNameRemoval && returnBool {
 			metric = metric.DropReserved(schema.IsMetadataLabel)
@@ -2972,6 +2970,10 @@ func (ev *evaluator) VectorBinop(op parser.ItemType, lhs, rhs Vector, matching *
 			insertedSigs[insertSig] = struct{}{}
 		}
 
+		if !keep && !returnBool {
+			continue
+		}
+
 		enh.Out = append(enh.Out, Sample{
 			Metric:   metric,
 			F:        floatValue,
diff --git a/promql/promqltest/testdata/operators.test b/promql/promqltest/testdata/operators.test
index 0e779f192c..e570be9630 100644
--- a/promql/promqltest/testdata/operators.test
+++ b/promql/promqltest/testdata/operators.test
@@ -316,6 +316,27 @@ eval instant at 5m http_requests_histogram == http_requests_histogram
 eval instant at 5m http_requests_histogram != http_requests_histogram
     expect no_info
 
+clear
+
+# Check that we track many-to-one vector matching errors even when all but 0 or 1
+# series on the "many" side are filtered away.
+load 5m
+  many_side{label="foo",job="test"} 0
+  many_side{label="bar",job="test"} 1
+  one_side{job="test"} 1
+
+# Check 0 series surviving the filtering producing an error.
+eval instant at 0m many_side > on(job) one_side
+  expect fail
+
+# Check 1 series surviving the filtering producing an error.
+eval instant at 0m many_side >= on(job) one_side
+  expect fail
+
+# Check 2 series surviving the filtering producing an error.
+eval instant at 0m many_side <= on(job) one_side
+  expect fail
+
 # group_left/group_right.
 
 clear