From 6b4bf3f69749c6a27a538ce726fc2191703916f0 Mon Sep 17 00:00:00 2001 From: Clayton Peters Date: Fri, 2 Jul 2021 16:52:47 +0100 Subject: [PATCH 01/19] PromQL: Fix start and end keywords masking label and metric names This commit fixes an issue with the "at modifier" that introduced two new keywords: `start` and `end`. In grouping options and in metric names, these keywords took precedence over metric or label names, so that those metrics and labels could no longer be referenced. Signed-off-by: Clayton Peters --- promql/parser/generated_parser.y | 4 +- promql/parser/generated_parser.y.go | 531 ++++++++++++++-------------- promql/parser/parse_test.go | 4 +- 3 files changed, 274 insertions(+), 265 deletions(-) diff --git a/promql/parser/generated_parser.y b/promql/parser/generated_parser.y index 3f914e4ac3..75f147ee41 100644 --- a/promql/parser/generated_parser.y +++ b/promql/parser/generated_parser.y @@ -571,7 +571,7 @@ metric : metric_identifier label_set ; -metric_identifier: AVG | BOTTOMK | BY | COUNT | COUNT_VALUES | GROUP | IDENTIFIER | LAND | LOR | LUNLESS | MAX | METRIC_IDENTIFIER | MIN | OFFSET | QUANTILE | STDDEV | STDVAR | SUM | TOPK | WITHOUT; +metric_identifier: AVG | BOTTOMK | BY | COUNT | COUNT_VALUES | GROUP | IDENTIFIER | LAND | LOR | LUNLESS | MAX | METRIC_IDENTIFIER | MIN | OFFSET | QUANTILE | STDDEV | STDVAR | SUM | TOPK | WITHOUT | START | END; label_set : LEFT_BRACE label_set_list RIGHT_BRACE { $$ = labels.New($2...) } @@ -674,7 +674,7 @@ series_value : IDENTIFIER aggregate_op : AVG | BOTTOMK | COUNT | COUNT_VALUES | GROUP | MAX | MIN | QUANTILE | STDDEV | STDVAR | SUM | TOPK ; // inside of grouping options label names can be recognized as keywords by the lexer. This is a list of keywords that could also be a label name. -maybe_label : AVG | BOOL | BOTTOMK | BY | COUNT | COUNT_VALUES | GROUP | GROUP_LEFT | GROUP_RIGHT | IDENTIFIER | IGNORING | LAND | LOR | LUNLESS | MAX | METRIC_IDENTIFIER | MIN | OFFSET | ON | QUANTILE | STDDEV | STDVAR | SUM | TOPK; +maybe_label : AVG | BOOL | BOTTOMK | BY | COUNT | COUNT_VALUES | GROUP | GROUP_LEFT | GROUP_RIGHT | IDENTIFIER | IGNORING | LAND | LOR | LUNLESS | MAX | METRIC_IDENTIFIER | MIN | OFFSET | ON | QUANTILE | STDDEV | STDVAR | SUM | TOPK | START | END; unary_op : ADD | SUB; diff --git a/promql/parser/generated_parser.y.go b/promql/parser/generated_parser.y.go index e0c5ceac57..5a2aafe3de 100644 --- a/promql/parser/generated_parser.y.go +++ b/promql/parser/generated_parser.y.go @@ -203,252 +203,259 @@ var yyExca = [...]int{ -1, 1, 1, -1, -2, 0, - -1, 33, - 1, 128, - 10, 128, - 22, 128, + -1, 35, + 1, 130, + 10, 130, + 22, 130, -2, 0, - -1, 56, - 2, 140, - 15, 140, - 61, 140, - 67, 140, - -2, 96, - -1, 57, - 2, 141, - 15, 141, - 61, 141, - 67, 141, - -2, 97, -1, 58, 2, 142, 15, 142, 61, 142, 67, 142, - -2, 99, + -2, 96, -1, 59, 2, 143, 15, 143, 61, 143, 67, 143, - -2, 100, + -2, 97, -1, 60, 2, 144, 15, 144, 61, 144, 67, 144, - -2, 101, + -2, 99, -1, 61, 2, 145, 15, 145, 61, 145, 67, 145, - -2, 106, + -2, 100, -1, 62, 2, 146, 15, 146, 61, 146, 67, 146, - -2, 108, + -2, 101, -1, 63, 2, 147, 15, 147, 61, 147, 67, 147, - -2, 110, + -2, 106, -1, 64, 2, 148, 15, 148, 61, 148, 67, 148, - -2, 111, + -2, 108, -1, 65, 2, 149, 15, 149, 61, 149, 67, 149, - -2, 112, + -2, 110, -1, 66, 2, 150, 15, 150, 61, 150, 67, 150, - -2, 113, + -2, 111, -1, 67, 2, 151, 15, 151, 61, 151, 67, 151, + -2, 112, + -1, 68, + 2, 152, + 15, 152, + 61, 152, + 67, 152, + -2, 113, + -1, 69, + 2, 153, + 15, 153, + 61, 153, + 67, 153, -2, 114, - -1, 186, - 12, 193, - 13, 193, - 16, 193, - 17, 193, - 23, 193, - 26, 193, - 32, 193, - 33, 193, - 36, 193, - 42, 193, - 46, 193, - 47, 193, - 48, 193, - 49, 193, - 50, 193, - 51, 193, - 52, 193, - 53, 193, - 54, 193, - 55, 193, - 56, 193, - 57, 193, - 61, 193, - 65, 193, - 67, 193, + -1, 188, + 12, 197, + 13, 197, + 16, 197, + 17, 197, + 23, 197, + 26, 197, + 32, 197, + 33, 197, + 36, 197, + 42, 197, + 46, 197, + 47, 197, + 48, 197, + 49, 197, + 50, 197, + 51, 197, + 52, 197, + 53, 197, + 54, 197, + 55, 197, + 56, 197, + 57, 197, + 61, 197, + 65, 197, + 67, 197, + 70, 197, + 71, 197, -2, 0, - -1, 187, - 12, 193, - 13, 193, - 16, 193, - 17, 193, - 23, 193, - 26, 193, - 32, 193, - 33, 193, - 36, 193, - 42, 193, - 46, 193, - 47, 193, - 48, 193, - 49, 193, - 50, 193, - 51, 193, - 52, 193, - 53, 193, - 54, 193, - 55, 193, - 56, 193, - 57, 193, - 61, 193, - 65, 193, - 67, 193, + -1, 189, + 12, 197, + 13, 197, + 16, 197, + 17, 197, + 23, 197, + 26, 197, + 32, 197, + 33, 197, + 36, 197, + 42, 197, + 46, 197, + 47, 197, + 48, 197, + 49, 197, + 50, 197, + 51, 197, + 52, 197, + 53, 197, + 54, 197, + 55, 197, + 56, 197, + 57, 197, + 61, 197, + 65, 197, + 67, 197, + 70, 197, + 71, 197, -2, 0, - -1, 207, - 19, 191, + -1, 209, + 19, 195, -2, 0, - -1, 254, - 19, 192, + -1, 258, + 19, 196, -2, 0, } const yyPrivate = 57344 -const yyLast = 638 +const yyLast = 654 var yyAct = [...]int{ - 260, 35, 211, 138, 250, 249, 146, 110, 75, 99, - 98, 101, 144, 184, 6, 185, 123, 102, 140, 100, - 186, 187, 55, 145, 245, 141, 150, 149, 263, 244, - 49, 70, 103, 51, 22, 50, 150, 118, 161, 252, - 243, 52, 151, 239, 68, 264, 261, 112, 149, 203, - 18, 19, 151, 105, 20, 106, 238, 111, 139, 104, - 69, 242, 119, 240, 56, 57, 58, 59, 60, 61, - 62, 63, 64, 65, 66, 67, 176, 107, 101, 13, - 147, 148, 158, 24, 102, 30, 2, 3, 4, 5, - 97, 258, 103, 7, 208, 157, 257, 175, 207, 166, - 70, 152, 80, 81, 165, 162, 156, 159, 154, 256, - 155, 206, 31, 90, 91, 164, 141, 93, 94, 96, - 117, 183, 116, 174, 265, 182, 188, 189, 190, 191, + 264, 37, 213, 140, 254, 253, 148, 112, 77, 101, + 100, 146, 186, 103, 187, 188, 189, 6, 102, 104, + 125, 267, 248, 147, 57, 99, 151, 247, 120, 51, + 72, 105, 53, 22, 52, 152, 72, 163, 265, 256, + 54, 268, 249, 70, 152, 243, 151, 205, 246, 18, + 19, 153, 95, 20, 98, 107, 105, 108, 242, 71, + 153, 106, 121, 58, 59, 60, 61, 62, 63, 64, + 65, 66, 67, 68, 69, 178, 97, 103, 13, 149, + 150, 109, 24, 104, 30, 33, 142, 31, 32, 2, + 3, 4, 5, 143, 210, 262, 177, 7, 209, 143, + 261, 168, 269, 154, 114, 79, 167, 164, 158, 161, + 156, 208, 157, 260, 113, 78, 48, 166, 81, 34, + 244, 175, 73, 185, 1, 176, 141, 184, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, - 79, 95, 181, 202, 124, 125, 126, 127, 128, 129, - 130, 131, 132, 133, 134, 135, 136, 137, 255, 10, - 168, 77, 169, 153, 54, 141, 112, 9, 9, 72, - 141, 76, 32, 241, 204, 205, 111, 173, 34, 97, - 49, 70, 109, 51, 22, 50, 246, 171, 1, 247, - 248, 52, 81, 251, 68, 8, 253, 170, 172, 33, - 18, 19, 90, 91, 20, 97, 93, 46, 96, 45, - 69, 254, 44, 71, 56, 57, 58, 59, 60, 61, - 62, 63, 64, 65, 66, 67, 43, 77, 42, 13, - 95, 122, 93, 24, 96, 30, 41, 76, 40, 39, - 120, 259, 178, 74, 160, 38, 262, 49, 70, 180, - 51, 22, 50, 121, 149, 37, 95, 115, 52, 36, - 267, 68, 114, 150, 268, 47, 142, 18, 19, 179, - 78, 20, 177, 113, 209, 73, 143, 69, 53, 151, - 212, 56, 57, 58, 59, 60, 61, 62, 63, 64, - 65, 66, 67, 213, 167, 48, 13, 108, 0, 0, - 24, 0, 30, 223, 0, 0, 0, 229, 0, 0, - 0, 266, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 225, 226, 0, 0, 227, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 214, 216, 218, - 219, 220, 228, 230, 233, 234, 235, 236, 237, 213, - 0, 215, 217, 221, 222, 224, 231, 232, 0, 223, - 0, 0, 0, 229, 0, 0, 0, 210, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 225, - 226, 0, 0, 227, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 214, 216, 218, 219, 220, 228, 230, - 233, 234, 235, 236, 237, 0, 0, 215, 217, 221, - 222, 224, 231, 232, 17, 70, 0, 0, 22, 0, + 202, 203, 257, 47, 183, 204, 126, 127, 128, 129, + 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, + 160, 46, 170, 117, 171, 259, 8, 119, 116, 118, + 35, 155, 143, 159, 114, 245, 206, 207, 143, 115, + 36, 99, 51, 72, 113, 53, 22, 52, 250, 173, + 111, 251, 252, 54, 83, 255, 70, 10, 45, 172, + 174, 44, 18, 19, 92, 93, 20, 74, 95, 124, + 98, 56, 71, 258, 9, 9, 58, 59, 60, 61, + 62, 63, 64, 65, 66, 67, 68, 69, 215, 43, + 42, 13, 97, 79, 41, 24, 122, 30, 225, 162, + 31, 32, 231, 78, 40, 263, 270, 123, 180, 76, + 266, 39, 38, 49, 144, 182, 181, 80, 227, 228, + 151, 179, 229, 211, 271, 75, 145, 55, 272, 152, + 214, 169, 216, 218, 220, 221, 222, 230, 232, 235, + 236, 237, 238, 239, 50, 153, 217, 219, 223, 224, + 226, 233, 234, 110, 0, 0, 240, 241, 51, 72, + 0, 53, 22, 52, 0, 0, 0, 0, 0, 54, + 0, 0, 70, 0, 0, 0, 0, 0, 18, 19, + 0, 0, 20, 0, 0, 0, 0, 0, 71, 0, + 0, 0, 58, 59, 60, 61, 62, 63, 64, 65, + 66, 67, 68, 69, 215, 0, 0, 13, 0, 0, + 0, 24, 0, 30, 225, 0, 31, 32, 231, 0, + 0, 0, 212, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 227, 228, 0, 0, 229, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 216, 218, + 220, 221, 222, 230, 232, 235, 236, 237, 238, 239, + 0, 0, 217, 219, 223, 224, 226, 233, 234, 0, + 17, 72, 240, 241, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 18, 19, 0, 0, 20, 0, - 17, 31, 0, 0, 22, 0, 0, 0, 11, 12, - 14, 15, 16, 21, 23, 25, 26, 27, 28, 29, - 18, 19, 0, 13, 20, 0, 0, 24, 0, 30, - 0, 0, 0, 0, 11, 12, 14, 15, 16, 21, - 23, 25, 26, 27, 28, 29, 97, 0, 0, 13, - 0, 0, 163, 24, 0, 30, 0, 0, 80, 81, - 82, 0, 83, 84, 85, 86, 87, 88, 89, 90, - 91, 92, 0, 93, 94, 96, 0, 0, 0, 0, - 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 80, 81, 82, 95, 83, 84, - 85, 86, 87, 88, 89, 90, 91, 92, 0, 93, - 94, 96, 0, 0, 97, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 80, 81, 82, 0, - 83, 84, 85, 95, 87, 88, 89, 90, 91, 92, - 0, 93, 94, 96, 0, 0, 97, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 80, 81, - 82, 0, 83, 84, 0, 95, 87, 88, 0, 90, - 91, 92, 0, 93, 94, 96, 0, 0, 0, 0, + 18, 19, 0, 0, 20, 0, 17, 33, 0, 0, + 22, 0, 0, 0, 11, 12, 14, 15, 16, 21, + 23, 25, 26, 27, 28, 29, 18, 19, 0, 13, + 20, 0, 0, 24, 0, 30, 0, 0, 31, 32, + 11, 12, 14, 15, 16, 21, 23, 25, 26, 27, + 28, 29, 0, 0, 99, 13, 0, 0, 0, 24, + 165, 30, 0, 0, 31, 32, 82, 83, 84, 0, + 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, + 0, 95, 96, 98, 0, 0, 0, 0, 0, 0, + 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 82, 83, 84, 97, 85, 86, 87, 88, + 89, 90, 91, 92, 93, 94, 0, 95, 96, 98, + 0, 0, 99, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 82, 83, 84, 0, 85, 86, + 87, 97, 89, 90, 91, 92, 93, 94, 0, 95, + 96, 98, 0, 0, 99, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 82, 83, 84, 0, + 85, 86, 99, 97, 89, 90, 0, 92, 93, 94, + 0, 95, 96, 98, 82, 83, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 92, 93, 0, 0, 95, + 96, 98, 0, 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 95, + 0, 0, 0, 97, } var yyPact = [...]int{ - 12, 83, 428, 428, 168, 402, -1000, -1000, -1000, 99, + 15, 87, 424, 424, 170, 398, -1000, -1000, -1000, 72, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, 225, -1000, 138, -1000, 508, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 17, 77, - -1000, 235, -1000, 235, 87, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, 231, -1000, 116, -1000, 506, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - 164, -1000, -1000, 255, -1000, -1000, 118, -1000, 15, -1000, - -44, -44, -44, -44, -44, -44, -44, -44, -44, -44, - -44, -44, -44, -44, -44, 16, 10, 161, 77, -50, - -1000, 80, 80, 18, -1000, 472, 191, -1000, 97, -1000, - -1000, 158, -1000, -1000, 159, -1000, 74, -1000, 237, 235, - -1000, -51, -42, -1000, 235, 235, 235, 235, 235, 235, - 235, 235, 235, 235, 235, 235, 235, 235, -1000, 107, - -1000, -1000, -1000, 34, -1000, -1000, -1000, -1000, -1000, -1000, - 31, 31, 92, -1000, -1000, -1000, -1000, 347, -1000, -1000, - 36, -1000, 508, -1000, -1000, 45, -1000, 38, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 5, 0, - -1000, -1000, -1000, 165, 80, 80, 80, 80, 191, 76, - 76, 76, 572, 540, 76, 76, 572, 191, 191, 76, - 191, 165, -1000, 19, -1000, -1000, -1000, 156, -1000, 89, + 16, 41, -1000, 286, -1000, 286, 23, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, 172, -1000, -1000, 161, -1000, -1000, 165, -1000, + 6, -1000, -40, -40, -40, -40, -40, -40, -40, -40, + -40, -40, -40, -40, -40, -40, -40, 84, 9, 169, + 41, -48, -1000, 158, 158, 17, -1000, 470, 11, -1000, + 99, -1000, -1000, 160, -1000, -1000, 103, -1000, 73, -1000, + 243, 286, -1000, -52, -47, -1000, 286, 286, 286, 286, + 286, 286, 286, 286, 286, 286, 286, 286, 286, 286, + -1000, 90, -1000, -1000, -1000, 32, -1000, -1000, -1000, -1000, + -1000, -1000, 29, 29, 92, -1000, -1000, -1000, -1000, 342, + -1000, -1000, 38, -1000, 506, -1000, -1000, 102, -1000, 25, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -2, 18, -1000, -1000, -1000, 167, 158, 158, 158, 158, + 11, 588, 588, 588, 570, 538, 588, 588, 570, 11, + 11, 588, 11, 167, -1000, 19, -1000, -1000, -1000, 163, + -1000, 93, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 235, - -1000, -1000, -1000, -1000, 29, 29, 4, -1000, -1000, -1000, - -1000, -1000, -1000, 26, 122, -1000, -1000, 291, -1000, 508, - -1000, -1000, -1000, 29, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, 286, -1000, -1000, -1000, -1000, 21, 21, + -3, -1000, -1000, -1000, -1000, -1000, -1000, 22, 100, -1000, + -1000, 226, -1000, 506, -1000, -1000, -1000, 21, -1000, -1000, + -1000, -1000, -1000, } var yyPgo = [...]int{ - 0, 297, 7, 295, 2, 294, 280, 164, 278, 276, - 159, 275, 195, 8, 274, 4, 5, 272, 270, 0, - 23, 269, 6, 266, 265, 259, 10, 62, 255, 253, - 1, 245, 244, 9, 240, 22, 239, 238, 236, 231, - 228, 226, 212, 209, 207, 3, 196, 188, 172, + 0, 293, 7, 284, 2, 271, 270, 211, 267, 266, + 197, 265, 166, 8, 263, 4, 5, 261, 257, 0, + 23, 256, 6, 254, 253, 252, 10, 62, 251, 247, + 1, 244, 239, 9, 236, 24, 234, 230, 229, 209, + 201, 198, 161, 143, 116, 3, 142, 124, 119, } var yyR1 = [...]int{ @@ -463,15 +470,15 @@ var yyR1 = [...]int{ 43, 44, 44, 44, 35, 35, 35, 1, 1, 1, 2, 2, 2, 2, 12, 12, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, - 7, 7, 7, 7, 7, 7, 10, 10, 10, 10, - 11, 11, 11, 13, 13, 13, 13, 48, 18, 18, - 18, 18, 17, 17, 17, 17, 17, 21, 21, 21, - 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, + 7, 7, 7, 7, 7, 7, 7, 7, 10, 10, + 10, 10, 11, 11, 11, 13, 13, 13, 13, 48, + 18, 18, 18, 18, 17, 17, 17, 17, 17, 21, + 21, 21, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, - 6, 6, 6, 6, 6, 6, 8, 8, 5, 5, - 5, 5, 37, 20, 22, 22, 23, 23, 19, 45, - 41, 46, 46, 16, 16, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 8, 8, 5, 5, 5, 5, 37, 20, 22, 22, + 23, 23, 19, 45, 41, 46, 46, 16, 16, } var yyR2 = [...]int{ @@ -486,75 +493,77 @@ var yyR2 = [...]int{ 2, 2, 1, 1, 3, 4, 2, 3, 1, 2, 3, 3, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 3, 4, 2, 0, - 3, 1, 2, 3, 3, 2, 1, 2, 0, 3, - 2, 1, 1, 3, 1, 3, 4, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 3, 4, + 2, 0, 3, 1, 2, 3, 3, 2, 1, 2, + 0, 3, 2, 1, 1, 3, 1, 3, 4, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 2, 2, 1, 1, 1, 1, - 1, 0, 1, 0, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, + 1, 1, 1, 1, 1, 0, 1, 0, 1, } var yyChk = [...]int{ -1000, -47, 74, 75, 76, 77, 2, 10, -12, -7, -10, 46, 47, 61, 48, 49, 50, 12, 32, 33, 36, 51, 16, 52, 65, 53, 54, 55, 56, 57, - 67, 13, -48, -12, 10, -30, -25, -28, -31, -36, - -37, -38, -40, -41, -42, -43, -44, -24, -3, 12, - 17, 15, 23, -8, -7, -35, 46, 47, 48, 49, - 50, 51, 52, 53, 54, 55, 56, 57, 26, 42, - 13, -44, -10, -11, 18, -13, 12, 2, -18, 2, - 26, 27, 28, 30, 31, 32, 33, 34, 35, 36, - 37, 38, 39, 41, 42, 65, 43, 14, -26, -33, - 2, 61, 67, 15, -33, -30, -30, -35, -1, 18, - -2, 12, 2, 18, 7, 2, 4, 2, 22, -27, - -34, -29, -39, 60, -27, -27, -27, -27, -27, -27, - -27, -27, -27, -27, -27, -27, -27, -27, -45, 42, - 2, 9, -23, -9, 2, -20, -22, 70, 71, 17, - 26, 42, -45, 2, -33, -26, -15, 15, 2, -15, - -32, 20, -30, 20, 18, 7, 2, -5, 2, 4, - 39, 29, 40, 18, -13, 23, 2, -17, 5, -21, - 12, -20, -22, -30, 64, 66, 62, 63, -30, -30, + 67, 70, 71, 13, -48, -12, 10, -30, -25, -28, + -31, -36, -37, -38, -40, -41, -42, -43, -44, -24, + -3, 12, 17, 15, 23, -8, -7, -35, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, + 26, 42, 13, -44, -10, -11, 18, -13, 12, 2, + -18, 2, 26, 27, 28, 30, 31, 32, 33, 34, + 35, 36, 37, 38, 39, 41, 42, 65, 43, 14, + -26, -33, 2, 61, 67, 15, -33, -30, -30, -35, + -1, 18, -2, 12, 2, 18, 7, 2, 4, 2, + 22, -27, -34, -29, -39, 60, -27, -27, -27, -27, + -27, -27, -27, -27, -27, -27, -27, -27, -27, -27, + -45, 42, 2, 9, -23, -9, 2, -20, -22, 70, + 71, 17, 26, 42, -45, 2, -33, -26, -15, 15, + 2, -15, -32, 20, -30, 20, 18, 7, 2, -5, + 2, 4, 39, 29, 40, 18, -13, 23, 2, -17, + 5, -21, 12, -20, -22, -30, 64, 66, 62, 63, -30, -30, -30, -30, -30, -30, -30, -30, -30, -30, - -30, -30, -45, 15, -20, -20, 19, 6, 2, -14, - 20, -4, -6, 2, 46, 60, 47, 61, 48, 49, - 50, 62, 63, 12, 64, 32, 33, 36, 51, 16, - 52, 65, 66, 53, 54, 55, 56, 57, 20, 7, - 18, -2, 23, 2, 24, 24, -22, -15, -15, -16, - -15, -16, 20, -46, -45, 2, 20, 7, 2, -30, - -19, 17, -19, 24, 19, 2, 20, -4, -19, + -30, -30, -30, -30, -45, 15, -20, -20, 19, 6, + 2, -14, 20, -4, -6, 2, 46, 60, 47, 61, + 48, 49, 50, 62, 63, 12, 64, 32, 33, 36, + 51, 16, 52, 65, 66, 53, 54, 55, 56, 57, + 70, 71, 20, 7, 18, -2, 23, 2, 24, 24, + -22, -15, -15, -16, -15, -16, 20, -46, -45, 2, + 20, 7, 2, -30, -19, 17, -19, 24, 19, 2, + 20, -4, -19, } var yyDef = [...]int{ - 0, -2, 119, 119, 0, 0, 7, 6, 1, 119, + 0, -2, 121, 121, 0, 0, 7, 6, 1, 121, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, - 115, 0, 2, -2, 3, 4, 8, 9, 10, 11, - 12, 13, 14, 15, 16, 17, 18, 19, 0, 102, - 182, 0, 190, 0, 82, 83, -2, -2, -2, -2, - -2, -2, -2, -2, -2, -2, -2, -2, 176, 177, - 0, 5, 94, 0, 118, 121, 0, 126, 127, 131, - 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, - 42, 42, 42, 42, 42, 0, 0, 0, 0, 22, - 23, 0, 0, 0, 59, 0, 80, 81, 0, 86, - 88, 0, 93, 116, 0, 122, 0, 125, 130, 0, - 41, 46, 47, 43, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, - 68, 189, 69, 0, 71, 186, 187, 72, 73, 183, - 0, 0, 0, 79, 20, 21, 24, 0, 53, 25, - 0, 61, 63, 65, 84, 0, 89, 0, 92, 178, - 179, 180, 181, 117, 120, 123, 124, 129, 132, 134, - 137, 138, 139, 26, 0, 0, -2, -2, 27, 28, - 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, - 39, 40, 67, 0, 184, 185, 74, -2, 78, 0, - 52, 55, 57, 58, 152, 153, 154, 155, 156, 157, + 115, 116, 117, 0, 2, -2, 3, 4, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + 0, 102, 186, 0, 194, 0, 82, 83, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + 180, 181, 0, 5, 94, 0, 120, 123, 0, 128, + 129, 133, 42, 42, 42, 42, 42, 42, 42, 42, + 42, 42, 42, 42, 42, 42, 42, 0, 0, 0, + 0, 22, 23, 0, 0, 0, 59, 0, 80, 81, + 0, 86, 88, 0, 93, 118, 0, 124, 0, 127, + 132, 0, 41, 46, 47, 43, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 66, 0, 68, 193, 69, 0, 71, 190, 191, 72, + 73, 187, 0, 0, 0, 79, 20, 21, 24, 0, + 53, 25, 0, 61, 63, 65, 84, 0, 89, 0, + 92, 182, 183, 184, 185, 119, 122, 125, 126, 131, + 134, 136, 139, 140, 141, 26, 0, 0, -2, -2, + 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, + 37, 38, 39, 40, 67, 0, 188, 189, 74, -2, + 78, 0, 52, 55, 57, 58, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, - 168, 169, 170, 171, 172, 173, 174, 175, 60, 64, - 85, 87, 90, 91, 0, 0, 0, 44, 45, 48, - 194, 49, 70, 0, -2, 77, 50, 0, 56, 62, - 133, 188, 135, 0, 75, 76, 51, 54, 136, + 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, + 178, 179, 60, 64, 85, 87, 90, 91, 0, 0, + 0, 44, 45, 48, 198, 49, 70, 0, -2, 77, + 50, 0, 56, 62, 135, 192, 137, 0, 75, 76, + 51, 54, 138, } var yyTok1 = [...]int{ @@ -1478,77 +1487,77 @@ yydefault: { yyVAL.labels = yyDollar[1].labels } - case 116: + case 118: yyDollar = yyS[yypt-3 : yypt+1] //line generated_parser.y:577 { yyVAL.labels = labels.New(yyDollar[2].labels...) } - case 117: + case 119: yyDollar = yyS[yypt-4 : yypt+1] //line generated_parser.y:579 { yyVAL.labels = labels.New(yyDollar[2].labels...) } - case 118: + case 120: yyDollar = yyS[yypt-2 : yypt+1] //line generated_parser.y:581 { yyVAL.labels = labels.New() } - case 119: + case 121: yyDollar = yyS[yypt-0 : yypt+1] //line generated_parser.y:583 { yyVAL.labels = labels.New() } - case 120: + case 122: yyDollar = yyS[yypt-3 : yypt+1] //line generated_parser.y:587 { yyVAL.labels = append(yyDollar[1].labels, yyDollar[3].label) } - case 121: + case 123: yyDollar = yyS[yypt-1 : yypt+1] //line generated_parser.y:589 { yyVAL.labels = []labels.Label{yyDollar[1].label} } - case 122: + case 124: yyDollar = yyS[yypt-2 : yypt+1] //line generated_parser.y:591 { yylex.(*parser).unexpected("label set", "\",\" or \"}\"") yyVAL.labels = yyDollar[1].labels } - case 123: + case 125: yyDollar = yyS[yypt-3 : yypt+1] //line generated_parser.y:596 { yyVAL.label = labels.Label{Name: yyDollar[1].item.Val, Value: yylex.(*parser).unquoteString(yyDollar[3].item.Val)} } - case 124: + case 126: yyDollar = yyS[yypt-3 : yypt+1] //line generated_parser.y:598 { yylex.(*parser).unexpected("label set", "string") yyVAL.label = labels.Label{} } - case 125: + case 127: yyDollar = yyS[yypt-2 : yypt+1] //line generated_parser.y:600 { yylex.(*parser).unexpected("label set", "\"=\"") yyVAL.label = labels.Label{} } - case 126: + case 128: yyDollar = yyS[yypt-1 : yypt+1] //line generated_parser.y:602 { yylex.(*parser).unexpected("label set", "identifier or \"}\"") yyVAL.label = labels.Label{} } - case 127: + case 129: yyDollar = yyS[yypt-2 : yypt+1] //line generated_parser.y:610 { @@ -1557,38 +1566,38 @@ yydefault: values: yyDollar[2].series, } } - case 128: + case 130: yyDollar = yyS[yypt-0 : yypt+1] //line generated_parser.y:619 { yyVAL.series = []SequenceValue{} } - case 129: + case 131: yyDollar = yyS[yypt-3 : yypt+1] //line generated_parser.y:621 { yyVAL.series = append(yyDollar[1].series, yyDollar[3].series...) } - case 130: + case 132: yyDollar = yyS[yypt-2 : yypt+1] //line generated_parser.y:623 { yyVAL.series = yyDollar[1].series } - case 131: + case 133: yyDollar = yyS[yypt-1 : yypt+1] //line generated_parser.y:625 { yylex.(*parser).unexpected("series values", "") yyVAL.series = nil } - case 132: + case 134: yyDollar = yyS[yypt-1 : yypt+1] //line generated_parser.y:629 { yyVAL.series = []SequenceValue{{Omitted: true}} } - case 133: + case 135: yyDollar = yyS[yypt-3 : yypt+1] //line generated_parser.y:631 { @@ -1597,13 +1606,13 @@ yydefault: yyVAL.series = append(yyVAL.series, SequenceValue{Omitted: true}) } } - case 134: + case 136: yyDollar = yyS[yypt-1 : yypt+1] //line generated_parser.y:638 { yyVAL.series = []SequenceValue{{Value: yyDollar[1].float}} } - case 135: + case 137: yyDollar = yyS[yypt-3 : yypt+1] //line generated_parser.y:640 { @@ -1612,7 +1621,7 @@ yydefault: yyVAL.series = append(yyVAL.series, SequenceValue{Value: yyDollar[1].float}) } } - case 136: + case 138: yyDollar = yyS[yypt-4 : yypt+1] //line generated_parser.y:647 { @@ -1622,7 +1631,7 @@ yydefault: yyDollar[1].float += yyDollar[2].float } } - case 137: + case 139: yyDollar = yyS[yypt-1 : yypt+1] //line generated_parser.y:657 { @@ -1631,7 +1640,7 @@ yydefault: } yyVAL.float = math.Float64frombits(value.StaleNaN) } - case 182: + case 186: yyDollar = yyS[yypt-1 : yypt+1] //line generated_parser.y:688 { @@ -1640,25 +1649,25 @@ yydefault: PosRange: yyDollar[1].item.PositionRange(), } } - case 183: + case 187: yyDollar = yyS[yypt-1 : yypt+1] //line generated_parser.y:696 { yyVAL.float = yylex.(*parser).number(yyDollar[1].item.Val) } - case 184: + case 188: yyDollar = yyS[yypt-2 : yypt+1] //line generated_parser.y:698 { yyVAL.float = yyDollar[2].float } - case 185: + case 189: yyDollar = yyS[yypt-2 : yypt+1] //line generated_parser.y:699 { yyVAL.float = -yyDollar[2].float } - case 188: + case 192: yyDollar = yyS[yypt-1 : yypt+1] //line generated_parser.y:705 { @@ -1668,7 +1677,7 @@ yydefault: yylex.(*parser).addParseErrf(yyDollar[1].item.PositionRange(), "invalid repetition in series values: %s", err) } } - case 189: + case 193: yyDollar = yyS[yypt-1 : yypt+1] //line generated_parser.y:715 { @@ -1678,7 +1687,7 @@ yydefault: yylex.(*parser).addParseErr(yyDollar[1].item.PositionRange(), err) } } - case 190: + case 194: yyDollar = yyS[yypt-1 : yypt+1] //line generated_parser.y:726 { @@ -1687,13 +1696,13 @@ yydefault: PosRange: yyDollar[1].item.PositionRange(), } } - case 191: + case 195: yyDollar = yyS[yypt-0 : yypt+1] //line generated_parser.y:739 { yyVAL.duration = 0 } - case 193: + case 197: yyDollar = yyS[yypt-0 : yypt+1] //line generated_parser.y:743 { diff --git a/promql/parser/parse_test.go b/promql/parser/parse_test.go index 9b3a8c84b5..a2f9967918 100644 --- a/promql/parser/parse_test.go +++ b/promql/parser/parse_test.go @@ -3172,11 +3172,11 @@ var testExpr = []struct { }, { input: `start()`, fail: true, - errMsg: `1:1: parse error: unexpected "start"`, + errMsg: `1:6: parse error: unexpected "("`, }, { input: `end()`, fail: true, - errMsg: `1:1: parse error: unexpected "end"`, + errMsg: `1:4: parse error: unexpected "("`, }, } From e7d55acbf9e14b93abfb1f257ffa953440274e80 Mon Sep 17 00:00:00 2001 From: Clayton Peters Date: Tue, 27 Jul 2021 15:19:59 +0100 Subject: [PATCH 02/19] Add in additional tests for metrics and/or labels called start/end. Signed-off-by: Clayton Peters --- promql/parser/parse_test.go | 112 ++++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) diff --git a/promql/parser/parse_test.go b/promql/parser/parse_test.go index a2f9967918..b73dfd22a9 100644 --- a/promql/parser/parse_test.go +++ b/promql/parser/parse_test.go @@ -3178,6 +3178,118 @@ var testExpr = []struct { fail: true, errMsg: `1:4: parse error: unexpected "("`, }, + // Check that start and end functions do not mask metrics. + { + input: `start`, + expected: &VectorSelector{ + Name: "start", + LabelMatchers: []*labels.Matcher{ + MustLabelMatcher(labels.MatchEqual, model.MetricNameLabel, "start"), + }, + PosRange: PositionRange{ + Start: 0, + End: 5, + }, + }, + }, { + input: `end`, + expected: &VectorSelector{ + Name: "end", + LabelMatchers: []*labels.Matcher{ + MustLabelMatcher(labels.MatchEqual, model.MetricNameLabel, "end"), + }, + PosRange: PositionRange{ + Start: 0, + End: 3, + }, + }, + }, { + input: `start{end="foo"}`, + expected: &VectorSelector{ + Name: "start", + LabelMatchers: []*labels.Matcher{ + MustLabelMatcher(labels.MatchEqual, "end", "foo"), + MustLabelMatcher(labels.MatchEqual, model.MetricNameLabel, "start"), + }, + PosRange: PositionRange{ + Start: 0, + End: 16, + }, + }, + }, { + input: `end{start="foo"}`, + expected: &VectorSelector{ + Name: "end", + LabelMatchers: []*labels.Matcher{ + MustLabelMatcher(labels.MatchEqual, "start", "foo"), + MustLabelMatcher(labels.MatchEqual, model.MetricNameLabel, "end"), + }, + PosRange: PositionRange{ + Start: 0, + End: 16, + }, + }, + }, { + input: `foo unless on(start) bar`, + expected: &BinaryExpr{ + Op: LUNLESS, + LHS: &VectorSelector{ + Name: "foo", + LabelMatchers: []*labels.Matcher{ + MustLabelMatcher(labels.MatchEqual, model.MetricNameLabel, "foo"), + }, + PosRange: PositionRange{ + Start: 0, + End: 3, + }, + }, + RHS: &VectorSelector{ + Name: "bar", + LabelMatchers: []*labels.Matcher{ + MustLabelMatcher(labels.MatchEqual, model.MetricNameLabel, "bar"), + }, + PosRange: PositionRange{ + Start: 21, + End: 24, + }, + }, + VectorMatching: &VectorMatching{ + Card: CardManyToMany, + MatchingLabels: []string{"start"}, + On: true, + }, + }, + }, { + input: `foo unless on(end) bar`, + expected: &BinaryExpr{ + Op: LUNLESS, + LHS: &VectorSelector{ + Name: "foo", + LabelMatchers: []*labels.Matcher{ + MustLabelMatcher(labels.MatchEqual, model.MetricNameLabel, "foo"), + }, + PosRange: PositionRange{ + Start: 0, + End: 3, + }, + }, + RHS: &VectorSelector{ + Name: "bar", + LabelMatchers: []*labels.Matcher{ + MustLabelMatcher(labels.MatchEqual, model.MetricNameLabel, "bar"), + }, + PosRange: PositionRange{ + Start: 19, + End: 22, + }, + }, + VectorMatching: &VectorMatching{ + Card: CardManyToMany, + MatchingLabels: []string{"end"}, + On: true, + }, + }, + }, } func makeInt64Pointer(val int64) *int64 { From b2f4bde6a07bdf5e189066a5a6cd43cca54d3572 Mon Sep 17 00:00:00 2001 From: Frederic Branczyk Date: Thu, 29 Jul 2021 10:16:54 +0200 Subject: [PATCH 03/19] *: Cut 2.29.0-rc.0 Signed-off-by: Frederic Branczyk --- CHANGELOG.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d61f6e5609..bdcff14f22 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,32 @@ +## 2.29.0-rc.0 / 2021-07-29 + +* [CHANGE] Promote `--storage.tsdb.allow-overlapping-blocks` flag to stable. #9117 +* [CHANGE] Promote `--storage.tsdb.retention.size` flag to stable. #9004 +* [FEATURE] Add Kuma service discovery. #8844 +* [FEATURE] Add `present_over_time` PromQL function. #9097 +* [FEATURE] Allow configuring exemplar storage via file and make it reloadable. #8974 +* [FEATURE] UI: Allow selecting time range with mouse drag. #8977 +* [FEATURE] promtool: Add feature flags flag `--enable-feature`. #8958 +* [FEATURE] promtool: Add file_sd file validation. #8950 +* [ENHANCEMENT] Reduce blocking of outgoing remote write requests from series garbage collection. #9109 +* [ENHANCEMENT] Improve write-ahead-log decoding performance. #9106 +* [ENHANCEMENT] Improve append performance in TSDB by reducing mutexes usage. #9061 +* [ENHANCEMENT] Allow configuring `max_samples_per_send` for remote write metadata. #8959 +* [ENHANCEMENT] Add `__meta_gce_interface_ipv4_` meta label to GCE discovery. #8978 +* [ENHANCEMENT] Add `__meta_ec2_availability_zone_id` meta label to EC2 discovery. #8978 +* [ENHANCEMENT] Add `__meta_azure_machine_computer_name` meta label to Azure discovery. #9112 +* [ENHANCEMENT] Add `__meta_hetzner_hcloud_labelpresent_` meta label to Hetzner discovery. #9028 +* [ENHANCEMENT] promtool: Add compaction efficiency to `promtool tsdb analyze` reports. #8940 +* [ENHANCEMENT] promtool: Allow configuring max block duration for backfilling via `--max-block-duration` flag. #8919 +* [ENHANCEMENT] UI: Add sorting and filtering to flags page. #8988 +* [ENHANCEMENT] UI: Improve alerts page rendering performance. #9005 +* [BUGFIX] Log when total symbol size exceeds 2^32 bytes, causing compaction to fail, and skip compaction. #9104 +* [BUGFIX] Fix incorrect `target_limit` reloading of zero value. #9120 +* [BUGFIX] Fix head GC and pending readers race condition. #9081 +* [BUGFIX] Fix timestamp handling in OpenMetrics parser. #9008 +* [BUGFIX] Fix potential duplicate metrics in `/federate` endpoint when specifying multiple matchers. #8885 +* [BUGFIX] Fix server configuration and validation for authentication via client cert. #9123 + ## 2.28.1 / 2021-07-01 * [BUGFIX]: HTTP SD: Allow `charset` specification in `Content-Type` header. #8981 From c9ae5f5ebf55200251ce9c8e42a65b4b00a549ee Mon Sep 17 00:00:00 2001 From: Frederic Branczyk Date: Fri, 30 Jul 2021 14:02:08 +0200 Subject: [PATCH 04/19] VERSION: bump to 2.29.0-rc.0 Signed-off-by: Frederic Branczyk --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 90efbd4e31..8a41e8cda9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.28.0 +2.29.0-rc.0 From daaff31e6576b413cdc1522c80371403b541d655 Mon Sep 17 00:00:00 2001 From: Julien Pivotto Date: Mon, 2 Aug 2021 11:08:55 +0200 Subject: [PATCH 05/19] Remove experimental wording on size-based retention Followup of #9004 Signed-off-by: Julien Pivotto --- cmd/prometheus/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go index 94b63882cc..c684575f2a 100644 --- a/cmd/prometheus/main.go +++ b/cmd/prometheus/main.go @@ -255,7 +255,7 @@ func main() { a.Flag("storage.tsdb.retention.time", "How long to retain samples in storage. When this flag is set it overrides \"storage.tsdb.retention\". If neither this flag nor \"storage.tsdb.retention\" nor \"storage.tsdb.retention.size\" is set, the retention time defaults to "+defaultRetentionString+". Units Supported: y, w, d, h, m, s, ms."). SetValue(&newFlagRetentionDuration) - a.Flag("storage.tsdb.retention.size", "Maximum number of bytes that can be stored for blocks. A unit is required, supported units: B, KB, MB, GB, TB, PB, EB. Ex: \"512MB\". This flag is experimental and can be changed in future releases."). + a.Flag("storage.tsdb.retention.size", "Maximum number of bytes that can be stored for blocks. A unit is required, supported units: B, KB, MB, GB, TB, PB, EB. Ex: \"512MB\"."). BytesVar(&cfg.tsdb.MaxBytes) a.Flag("storage.tsdb.no-lockfile", "Do not create lockfile in data directory."). From 330cc33d63534c0a439b74c709cf1daab9b2c6e3 Mon Sep 17 00:00:00 2001 From: George Brighton Date: Mon, 2 Aug 2021 12:45:49 +0100 Subject: [PATCH 06/19] Fix PR reference in changelog Signed-off-by: George Brighton --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bdcff14f22..ae8470af3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ * [ENHANCEMENT] Improve append performance in TSDB by reducing mutexes usage. #9061 * [ENHANCEMENT] Allow configuring `max_samples_per_send` for remote write metadata. #8959 * [ENHANCEMENT] Add `__meta_gce_interface_ipv4_` meta label to GCE discovery. #8978 -* [ENHANCEMENT] Add `__meta_ec2_availability_zone_id` meta label to EC2 discovery. #8978 +* [ENHANCEMENT] Add `__meta_ec2_availability_zone_id` meta label to EC2 discovery. #8896 * [ENHANCEMENT] Add `__meta_azure_machine_computer_name` meta label to Azure discovery. #9112 * [ENHANCEMENT] Add `__meta_hetzner_hcloud_labelpresent_` meta label to Hetzner discovery. #9028 * [ENHANCEMENT] promtool: Add compaction efficiency to `promtool tsdb analyze` reports. #8940 From a715dd84ecf1eec7fa562265465171be8ab2ff5d Mon Sep 17 00:00:00 2001 From: George Brighton Date: Mon, 2 Aug 2021 13:55:54 +0100 Subject: [PATCH 07/19] Describe EC2 availability zone IDs at most once per refresh (#9142) Signed-off-by: George Brighton --- discovery/aws/ec2.go | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/discovery/aws/ec2.go b/discovery/aws/ec2.go index d2ad9736ad..351f8c7db1 100644 --- a/discovery/aws/ec2.go +++ b/discovery/aws/ec2.go @@ -194,24 +194,35 @@ func (d *EC2Discovery) ec2Client(ctx context.Context) (*ec2.EC2, error) { return d.ec2, nil } -func (d *EC2Discovery) azID(ctx context.Context, az string) (string, error) { +func (d *EC2Discovery) azID(ctx context.Context, az string, azIDsRefreshAttempted *bool) (string, error) { if azID, ok := d.azToAZID[az]; ok { return azID, nil } + if !*azIDsRefreshAttempted { + *azIDsRefreshAttempted = true + if err := d.refreshAZIDs(ctx); err != nil { + return "", err + } + if azID, ok := d.azToAZID[az]; ok { + return azID, nil + } + } + + return "", fmt.Errorf("no availability zone ID mapping found for %s", az) +} + +func (d *EC2Discovery) refreshAZIDs(ctx context.Context) error { azs, err := d.ec2.DescribeAvailabilityZonesWithContext(ctx, &ec2.DescribeAvailabilityZonesInput{}) if err != nil { - return "", errors.Wrap(err, "could not describe availability zones") + // Likely due to missing ec2:DescribeAvailabilityZones. + return errors.Wrap(err, "could not describe availability zones") } d.azToAZID = make(map[string]string, len(azs.AvailabilityZones)) for _, az := range azs.AvailabilityZones { d.azToAZID[*az.ZoneName] = *az.ZoneId } - - if azID, ok := d.azToAZID[az]; ok { - return azID, nil - } - return "", fmt.Errorf("no availability zone ID mapping found for %s", az) + return nil } func (d *EC2Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { @@ -233,14 +244,14 @@ func (d *EC2Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error } input := &ec2.DescribeInstancesInput{Filters: filters} - + azIDsRefreshAttempted := false if err := ec2Client.DescribeInstancesPagesWithContext(ctx, input, func(p *ec2.DescribeInstancesOutput, lastPage bool) bool { for _, r := range p.Reservations { for _, inst := range r.Instances { if inst.PrivateIpAddress == nil { continue } - azID, err := d.azID(ctx, *inst.Placement.AvailabilityZone) + azID, err := d.azID(ctx, *inst.Placement.AvailabilityZone, &azIDsRefreshAttempted) if err != nil { level.Warn(d.logger).Log( "msg", "Unable to determine availability zone ID", From 2d8f9c6f61eddfba0c9aab05d4709e6ae5ba4a82 Mon Sep 17 00:00:00 2001 From: George Brighton Date: Mon, 2 Aug 2021 20:52:37 +0100 Subject: [PATCH 08/19] Describe EC2 availability zones at most once per SD load Closes #9142. Signed-off-by: George Brighton --- discovery/aws/ec2.go | 46 ++++++++++++++++---------------------------- 1 file changed, 17 insertions(+), 29 deletions(-) diff --git a/discovery/aws/ec2.go b/discovery/aws/ec2.go index 351f8c7db1..05a44e1a9a 100644 --- a/discovery/aws/ec2.go +++ b/discovery/aws/ec2.go @@ -194,29 +194,10 @@ func (d *EC2Discovery) ec2Client(ctx context.Context) (*ec2.EC2, error) { return d.ec2, nil } -func (d *EC2Discovery) azID(ctx context.Context, az string, azIDsRefreshAttempted *bool) (string, error) { - if azID, ok := d.azToAZID[az]; ok { - return azID, nil - } - - if !*azIDsRefreshAttempted { - *azIDsRefreshAttempted = true - if err := d.refreshAZIDs(ctx); err != nil { - return "", err - } - if azID, ok := d.azToAZID[az]; ok { - return azID, nil - } - } - - return "", fmt.Errorf("no availability zone ID mapping found for %s", az) -} - func (d *EC2Discovery) refreshAZIDs(ctx context.Context) error { azs, err := d.ec2.DescribeAvailabilityZonesWithContext(ctx, &ec2.DescribeAvailabilityZonesInput{}) if err != nil { - // Likely due to missing ec2:DescribeAvailabilityZones. - return errors.Wrap(err, "could not describe availability zones") + return err } d.azToAZID = make(map[string]string, len(azs.AvailabilityZones)) for _, az := range azs.AvailabilityZones { @@ -243,21 +224,23 @@ func (d *EC2Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error }) } + // Only refresh the AZ ID map if we have never been able to build one. + // Prometheus requires a reload if AWS adds a new AZ to the region. + if d.azToAZID == nil { + if err := d.refreshAZIDs(ctx); err != nil { + level.Warn(d.logger).Log( + "msg", "Unable to describe availability zones, ensure Prometheus has ec2:DescribeAvailabilityZones", + "err", err) + } + } + input := &ec2.DescribeInstancesInput{Filters: filters} - azIDsRefreshAttempted := false if err := ec2Client.DescribeInstancesPagesWithContext(ctx, input, func(p *ec2.DescribeInstancesOutput, lastPage bool) bool { for _, r := range p.Reservations { for _, inst := range r.Instances { if inst.PrivateIpAddress == nil { continue } - azID, err := d.azID(ctx, *inst.Placement.AvailabilityZone, &azIDsRefreshAttempted) - if err != nil { - level.Warn(d.logger).Log( - "msg", "Unable to determine availability zone ID", - "az", *inst.Placement.AvailabilityZone, - "err", err) - } labels := model.LabelSet{ ec2LabelInstanceID: model.LabelValue(*inst.InstanceId), @@ -282,9 +265,14 @@ func (d *EC2Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error labels[ec2LabelPublicIP] = model.LabelValue(*inst.PublicIpAddress) labels[ec2LabelPublicDNS] = model.LabelValue(*inst.PublicDnsName) } - labels[ec2LabelAMI] = model.LabelValue(*inst.ImageId) labels[ec2LabelAZ] = model.LabelValue(*inst.Placement.AvailabilityZone) + azID, ok := d.azToAZID[*inst.Placement.AvailabilityZone] + if !ok && d.azToAZID != nil { + level.Debug(d.logger).Log( + "msg", "Availability zone not found, try reloading Prometheus", + "az", *inst.Placement.AvailabilityZone) + } labels[ec2LabelAZID] = model.LabelValue(azID) labels[ec2LabelInstanceState] = model.LabelValue(*inst.State.Name) labels[ec2LabelInstanceType] = model.LabelValue(*inst.InstanceType) From 1f752b6910f2aecfb2e20846ec9f91951b1d502b Mon Sep 17 00:00:00 2001 From: George Brighton Date: Tue, 3 Aug 2021 03:32:31 +0100 Subject: [PATCH 09/19] Incorporate feedback Signed-off-by: George Brighton --- discovery/aws/ec2.go | 6 +++--- docs/configuration/configuration.md | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/discovery/aws/ec2.go b/discovery/aws/ec2.go index 05a44e1a9a..0b2bb316ba 100644 --- a/discovery/aws/ec2.go +++ b/discovery/aws/ec2.go @@ -228,8 +228,8 @@ func (d *EC2Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error // Prometheus requires a reload if AWS adds a new AZ to the region. if d.azToAZID == nil { if err := d.refreshAZIDs(ctx); err != nil { - level.Warn(d.logger).Log( - "msg", "Unable to describe availability zones, ensure Prometheus has ec2:DescribeAvailabilityZones", + level.Debug(d.logger).Log( + "msg", "Unable to describe availability zones", "err", err) } } @@ -270,7 +270,7 @@ func (d *EC2Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error azID, ok := d.azToAZID[*inst.Placement.AvailabilityZone] if !ok && d.azToAZID != nil { level.Debug(d.logger).Log( - "msg", "Availability zone not found, try reloading Prometheus", + "msg", "Availability zone not found", "az", *inst.Placement.AvailabilityZone) } labels[ec2LabelAZID] = model.LabelValue(azID) diff --git a/docs/configuration/configuration.md b/docs/configuration/configuration.md index e48ba6a0b4..d5dd52d05c 100644 --- a/docs/configuration/configuration.md +++ b/docs/configuration/configuration.md @@ -898,7 +898,7 @@ The following meta labels are available on targets during [relabeling](#relabel_ * `__meta_ec2_ami`: the EC2 Amazon Machine Image * `__meta_ec2_architecture`: the architecture of the instance * `__meta_ec2_availability_zone`: the availability zone in which the instance is running -* `__meta_ec2_availability_zone_id`: the [availability zone ID](https://docs.aws.amazon.com/ram/latest/userguide/working-with-az-ids.html) in which the instance is running +* `__meta_ec2_availability_zone_id`: the [availability zone ID](https://docs.aws.amazon.com/ram/latest/userguide/working-with-az-ids.html) in which the instance is running (requires `ec2:DescribeAvailabilityZones`) * `__meta_ec2_instance_id`: the EC2 instance ID * `__meta_ec2_instance_lifecycle`: the lifecycle of the EC2 instance, set only for 'spot' or 'scheduled' instances, absent otherwise * `__meta_ec2_instance_state`: the state of the EC2 instance From e6e7b08e6f8b3044ae016a179ba51b3b17bb3634 Mon Sep 17 00:00:00 2001 From: George Brighton Date: Tue, 3 Aug 2021 16:46:51 +0100 Subject: [PATCH 10/19] Integrate feedback Signed-off-by: George Brighton --- discovery/aws/ec2.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/discovery/aws/ec2.go b/discovery/aws/ec2.go index 0b2bb316ba..0bcfd05476 100644 --- a/discovery/aws/ec2.go +++ b/discovery/aws/ec2.go @@ -270,7 +270,7 @@ func (d *EC2Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error azID, ok := d.azToAZID[*inst.Placement.AvailabilityZone] if !ok && d.azToAZID != nil { level.Debug(d.logger).Log( - "msg", "Availability zone not found", + "msg", "Availability zone ID not found", "az", *inst.Placement.AvailabilityZone) } labels[ec2LabelAZID] = model.LabelValue(azID) From 4fdf25dda556aa6d115ee4afd8b4e823b0a0a846 Mon Sep 17 00:00:00 2001 From: Julien Pivotto Date: Tue, 3 Aug 2021 12:33:31 +0200 Subject: [PATCH 11/19] Add a compatibility note for macOS users. Signed-off-by: Julien Pivotto --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bdcff14f22..e055b2731f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ ## 2.29.0-rc.0 / 2021-07-29 +Note for macOS users: Due to [changes in the upcoming Go 1.17](https://tip.golang.org/doc/go1.17#darwin), +this is the last Prometheus release that supports macOS 10.12 Sierra. + * [CHANGE] Promote `--storage.tsdb.allow-overlapping-blocks` flag to stable. #9117 * [CHANGE] Promote `--storage.tsdb.retention.size` flag to stable. #9004 * [FEATURE] Add Kuma service discovery. #8844 From 868776b4d0ac854ffa676128fc5d0f568baab874 Mon Sep 17 00:00:00 2001 From: Frederic Branczyk Date: Wed, 4 Aug 2021 10:22:42 +0200 Subject: [PATCH 12/19] *: Cut v2.29.0-rc.1 Signed-off-by: Frederic Branczyk --- CHANGELOG.md | 3 ++- VERSION | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 69df7013a6..65c9dba40e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## 2.29.0-rc.0 / 2021-07-29 +## 2.29.0-rc.1 / 2021-08-04 Note for macOS users: Due to [changes in the upcoming Go 1.17](https://tip.golang.org/doc/go1.17#darwin), this is the last Prometheus release that supports macOS 10.12 Sierra. @@ -29,6 +29,7 @@ this is the last Prometheus release that supports macOS 10.12 Sierra. * [BUGFIX] Fix timestamp handling in OpenMetrics parser. #9008 * [BUGFIX] Fix potential duplicate metrics in `/federate` endpoint when specifying multiple matchers. #8885 * [BUGFIX] Fix server configuration and validation for authentication via client cert. #9123 +* [BUGFIX] Allow `start` and `end` again as label names in PromQL queries. They were disallowed since the introduction of @ timestamp feature. #9119 ## 2.28.1 / 2021-07-01 diff --git a/VERSION b/VERSION index 8a41e8cda9..4239816eed 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.29.0-rc.0 +2.29.0-rc.1 From 39a8db22eb03a4bc6aaab255c410ef82c396cc5d Mon Sep 17 00:00:00 2001 From: Austin Cawley-Edwards Date: Thu, 5 Aug 2021 19:29:15 -0400 Subject: [PATCH 13/19] Fix `kuma_sd` targetgroup reporting (#9157) * Bundle all xDS targets into a single group Signed-off-by: austin ce --- discovery/xds/kuma.go | 34 ++++---- discovery/xds/kuma_test.go | 156 ++++++++++++++----------------------- discovery/xds/xds.go | 17 ++-- discovery/xds/xds_test.go | 103 ++++++++++++++++++++---- 4 files changed, 170 insertions(+), 140 deletions(-) diff --git a/discovery/xds/kuma.go b/discovery/xds/kuma.go index d4071be9ca..77f9f0561b 100644 --- a/discovery/xds/kuma.go +++ b/discovery/xds/kuma.go @@ -27,7 +27,6 @@ import ( "google.golang.org/protobuf/types/known/anypb" "github.com/prometheus/prometheus/discovery" - "github.com/prometheus/prometheus/discovery/targetgroup" "github.com/prometheus/prometheus/util/osutil" "github.com/prometheus/prometheus/util/strutil" ) @@ -129,30 +128,27 @@ func (c *KumaSDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discover return NewKumaHTTPDiscovery(c, logger) } -func convertKumaV1MonitoringAssignment(assignment *MonitoringAssignment) *targetgroup.Group { +func convertKumaV1MonitoringAssignment(assignment *MonitoringAssignment) []model.LabelSet { commonLabels := convertKumaUserLabels(assignment.Labels) commonLabels[kumaMeshLabel] = model.LabelValue(assignment.Mesh) commonLabels[kumaServiceLabel] = model.LabelValue(assignment.Service) - var targetLabelSets []model.LabelSet + var targets []model.LabelSet - for _, target := range assignment.Targets { - targetLabels := convertKumaUserLabels(target.Labels) + for _, madsTarget := range assignment.Targets { + targetLabels := convertKumaUserLabels(madsTarget.Labels).Merge(commonLabels) - targetLabels[kumaDataplaneLabel] = model.LabelValue(target.Name) - targetLabels[model.InstanceLabel] = model.LabelValue(target.Name) - targetLabels[model.AddressLabel] = model.LabelValue(target.Address) - targetLabels[model.SchemeLabel] = model.LabelValue(target.Scheme) - targetLabels[model.MetricsPathLabel] = model.LabelValue(target.MetricsPath) + targetLabels[kumaDataplaneLabel] = model.LabelValue(madsTarget.Name) + targetLabels[model.AddressLabel] = model.LabelValue(madsTarget.Address) + targetLabels[model.InstanceLabel] = model.LabelValue(madsTarget.Name) + targetLabels[model.SchemeLabel] = model.LabelValue(madsTarget.Scheme) + targetLabels[model.MetricsPathLabel] = model.LabelValue(madsTarget.MetricsPath) - targetLabelSets = append(targetLabelSets, targetLabels) + targets = append(targets, targetLabels) } - return &targetgroup.Group{ - Labels: commonLabels, - Targets: targetLabelSets, - } + return targets } func convertKumaUserLabels(labels map[string]string) model.LabelSet { @@ -165,12 +161,12 @@ func convertKumaUserLabels(labels map[string]string) model.LabelSet { } // kumaMadsV1ResourceParser is an xds.resourceParser. -func kumaMadsV1ResourceParser(resources []*anypb.Any, typeURL string) ([]*targetgroup.Group, error) { +func kumaMadsV1ResourceParser(resources []*anypb.Any, typeURL string) ([]model.LabelSet, error) { if typeURL != KumaMadsV1ResourceTypeURL { return nil, errors.Errorf("recieved invalid typeURL for Kuma MADS v1 Resource: %s", typeURL) } - var groups []*targetgroup.Group + var targets []model.LabelSet for _, resource := range resources { assignment := &MonitoringAssignment{} @@ -179,10 +175,10 @@ func kumaMadsV1ResourceParser(resources []*anypb.Any, typeURL string) ([]*target return nil, err } - groups = append(groups, convertKumaV1MonitoringAssignment(assignment)) + targets = append(targets, convertKumaV1MonitoringAssignment(assignment)...) } - return groups, nil + return targets, nil } func NewKumaHTTPDiscovery(conf *KumaSDConfig, logger log.Logger) (discovery.Discoverer, error) { diff --git a/discovery/xds/kuma_test.go b/discovery/xds/kuma_test.go index 8db2ce443c..0de1b986d2 100644 --- a/discovery/xds/kuma_test.go +++ b/discovery/xds/kuma_test.go @@ -138,65 +138,47 @@ func TestKumaMadsV1ResourceParserValidResources(t *testing.T) { res, err := getKumaMadsV1DiscoveryResponse(testKumaMadsV1Resources...) require.NoError(t, err) - groups, err := kumaMadsV1ResourceParser(res.Resources, KumaMadsV1ResourceTypeURL) + targets, err := kumaMadsV1ResourceParser(res.Resources, KumaMadsV1ResourceTypeURL) require.NoError(t, err) - require.Len(t, groups, 3) + require.Len(t, targets, 3) - expectedGroup1 := &targetgroup.Group{ - Targets: []model.LabelSet{ - { - "__address__": "10.1.4.32:9090", - "__meta_kuma_label_commit_hash": "620506a88", - "__meta_kuma_dataplane": "prometheus-01", - "__metrics_path__": "/custom-metrics", - "__scheme__": "http", - "instance": "prometheus-01", - }, - { - "__address__": "10.1.4.33:9090", - "__meta_kuma_label_commit_hash": "3513bba00", - "__meta_kuma_dataplane": "prometheus-02", - "__metrics_path__": "", - "__scheme__": "http", - "instance": "prometheus-02", - }, - }, - Labels: model.LabelSet{ + expectedTargets := []model.LabelSet{ + { + "__address__": "10.1.4.32:9090", + "__metrics_path__": "/custom-metrics", + "__scheme__": "http", + "instance": "prometheus-01", "__meta_kuma_mesh": "metrics", "__meta_kuma_service": "prometheus", "__meta_kuma_label_team": "infra", "__meta_kuma_label_kuma_io_zone": "us-east-1", + "__meta_kuma_label_commit_hash": "620506a88", + "__meta_kuma_dataplane": "prometheus-01", }, - } - require.Equal(t, expectedGroup1, groups[0]) - - expectedGroup2 := &targetgroup.Group{ - Labels: model.LabelSet{ + { + "__address__": "10.1.4.33:9090", + "__metrics_path__": "", + "__scheme__": "http", + "instance": "prometheus-02", "__meta_kuma_mesh": "metrics", - "__meta_kuma_service": "grafana", + "__meta_kuma_service": "prometheus", "__meta_kuma_label_team": "infra", "__meta_kuma_label_kuma_io_zone": "us-east-1", + "__meta_kuma_label_commit_hash": "3513bba00", + "__meta_kuma_dataplane": "prometheus-02", + }, + { + "__address__": "10.1.1.1", + "__metrics_path__": "", + "__scheme__": "http", + "instance": "elasticsearch-01", + "__meta_kuma_mesh": "data", + "__meta_kuma_service": "elasticsearch", + "__meta_kuma_label_role": "ml", + "__meta_kuma_dataplane": "elasticsearch-01", }, } - require.Equal(t, expectedGroup2, groups[1]) - - expectedGroup3 := &targetgroup.Group{ - Targets: []model.LabelSet{ - { - "__address__": "10.1.1.1", - "__meta_kuma_label_role": "ml", - "__meta_kuma_dataplane": "elasticsearch-01", - "__metrics_path__": "", - "__scheme__": "http", - "instance": "elasticsearch-01", - }, - }, - Labels: model.LabelSet{ - "__meta_kuma_mesh": "data", - "__meta_kuma_service": "elasticsearch", - }, - } - require.Equal(t, expectedGroup3, groups[2]) + require.Equal(t, expectedTargets, targets) } func TestKumaMadsV1ResourceParserInvalidResources(t *testing.T) { @@ -262,66 +244,48 @@ tls_config: kd.poll(context.Background(), ch) groups := <-ch - require.Len(t, groups, 3) + require.Len(t, groups, 1) - expectedGroup1 := &targetgroup.Group{ - Source: "kuma", - Targets: []model.LabelSet{ - { - "__address__": "10.1.4.32:9090", - "__meta_kuma_label_commit_hash": "620506a88", - "__meta_kuma_dataplane": "prometheus-01", - "__metrics_path__": "/custom-metrics", - "__scheme__": "http", - "instance": "prometheus-01", - }, - { - "__address__": "10.1.4.33:9090", - "__meta_kuma_label_commit_hash": "3513bba00", - "__meta_kuma_dataplane": "prometheus-02", - "__metrics_path__": "", - "__scheme__": "http", - "instance": "prometheus-02", - }, - }, - Labels: model.LabelSet{ + targets := groups[0].Targets + require.Len(t, targets, 3) + + expectedTargets := []model.LabelSet{ + { + "__address__": "10.1.4.32:9090", + "__metrics_path__": "/custom-metrics", + "__scheme__": "http", + "instance": "prometheus-01", "__meta_kuma_mesh": "metrics", "__meta_kuma_service": "prometheus", "__meta_kuma_label_team": "infra", "__meta_kuma_label_kuma_io_zone": "us-east-1", + "__meta_kuma_label_commit_hash": "620506a88", + "__meta_kuma_dataplane": "prometheus-01", }, - } - require.Equal(t, expectedGroup1, groups[0]) - - expectedGroup2 := &targetgroup.Group{ - Source: "kuma", - Labels: model.LabelSet{ + { + "__address__": "10.1.4.33:9090", + "__metrics_path__": "", + "__scheme__": "http", + "instance": "prometheus-02", "__meta_kuma_mesh": "metrics", - "__meta_kuma_service": "grafana", + "__meta_kuma_service": "prometheus", "__meta_kuma_label_team": "infra", "__meta_kuma_label_kuma_io_zone": "us-east-1", + "__meta_kuma_label_commit_hash": "3513bba00", + "__meta_kuma_dataplane": "prometheus-02", + }, + { + "__address__": "10.1.1.1", + "__metrics_path__": "", + "__scheme__": "http", + "instance": "elasticsearch-01", + "__meta_kuma_mesh": "data", + "__meta_kuma_service": "elasticsearch", + "__meta_kuma_label_role": "ml", + "__meta_kuma_dataplane": "elasticsearch-01", }, } - require.Equal(t, expectedGroup2, groups[1]) - - expectedGroup3 := &targetgroup.Group{ - Source: "kuma", - Targets: []model.LabelSet{ - { - "__address__": "10.1.1.1", - "__meta_kuma_label_role": "ml", - "__meta_kuma_dataplane": "elasticsearch-01", - "__metrics_path__": "", - "__scheme__": "http", - "instance": "elasticsearch-01", - }, - }, - Labels: model.LabelSet{ - "__meta_kuma_mesh": "data", - "__meta_kuma_service": "elasticsearch", - }, - } - require.Equal(t, expectedGroup3, groups[2]) + require.Equal(t, expectedTargets, targets) // Should skip the next update. ctx, cancel := context.WithCancel(context.Background()) diff --git a/discovery/xds/xds.go b/discovery/xds/xds.go index f99e03da98..48bdbab02b 100644 --- a/discovery/xds/xds.go +++ b/discovery/xds/xds.go @@ -15,7 +15,6 @@ package xds import ( "context" - "github.com/prometheus/common/model" "time" v3 "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" @@ -23,6 +22,7 @@ import ( "github.com/go-kit/log/level" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/config" + "github.com/prometheus/common/model" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/reflect/protoreflect" @@ -95,7 +95,9 @@ var ( } ) -type resourceParser func(resources []*anypb.Any, typeUrl string) ([]*targetgroup.Group, error) +// resourceParser is a function that takes raw discovered objects and translates them into +// targetgroup.Group Targets. On error, no updates are sent to the scrape manager and the failure count is incremented. +type resourceParser func(resources []*anypb.Any, typeUrl string) ([]model.LabelSet, error) // fetchDiscovery implements long-polling via xDS Fetch REST-JSON. type fetchDiscovery struct { @@ -154,23 +156,18 @@ func (d *fetchDiscovery) poll(ctx context.Context, ch chan<- []*targetgroup.Grou return } - parsedGroups, err := d.parseResources(response.Resources, response.TypeUrl) + parsedTargets, err := d.parseResources(response.Resources, response.TypeUrl) if err != nil { level.Error(d.logger).Log("msg", "error parsing resources", "err", err) d.fetchFailuresCount.Inc() return } - for _, group := range parsedGroups { - group.Source = d.source - } + level.Debug(d.logger).Log("msg", "Updated to version", "version", response.VersionInfo, "targets", len(parsedTargets)) - level.Debug(d.logger).Log("msg", "updated to version", "version", response.VersionInfo, "groups", len(parsedGroups)) - - // Check the context before sending an update on the channel. select { case <-ctx.Done(): return - case ch <- parsedGroups: + case ch <- []*targetgroup.Group{{Source: d.source, Targets: parsedTargets}}: } } diff --git a/discovery/xds/xds_test.go b/discovery/xds/xds_test.go index 412cbda68f..c0cdd3e7b4 100644 --- a/discovery/xds/xds_test.go +++ b/discovery/xds/xds_test.go @@ -93,9 +93,9 @@ func createTestHTTPServer(t *testing.T, responder discoveryResponder) *httptest. })) } -func constantResourceParser(groups []*targetgroup.Group, err error) resourceParser { - return func(resources []*anypb.Any, typeUrl string) ([]*targetgroup.Group, error) { - return groups, err +func constantResourceParser(targets []model.LabelSet, err error) resourceParser { + return func(resources []*anypb.Any, typeUrl string) ([]model.LabelSet, error) { + return targets, err } } @@ -174,13 +174,16 @@ func TestPollingRefreshAttachesGroupMetadata(t *testing.T) { fetchDuration: testFetchDuration, fetchFailuresCount: testFetchFailuresCount, fetchSkipUpdateCount: testFetchSkipUpdateCount, - parseResources: constantResourceParser([]*targetgroup.Group{ - {}, + parseResources: constantResourceParser([]model.LabelSet{ { - Source: "a-custom-source", - Labels: model.LabelSet{ - "__meta_custom_xds_label": "a-value", - }, + "__meta_custom_xds_label": "a-value", + "__address__": "10.1.4.32:9090", + "instance": "prometheus-01", + }, + { + "__meta_custom_xds_label": "a-value", + "__address__": "10.1.5.32:9090", + "instance": "prometheus-02", }, }, nil), } @@ -189,13 +192,83 @@ func TestPollingRefreshAttachesGroupMetadata(t *testing.T) { groups := <-ch require.NotNil(t, groups) - require.Len(t, groups, 2) + require.Len(t, groups, 1) - for _, group := range groups { - require.Equal(t, source, group.Source) + group := groups[0] + require.Equal(t, source, group.Source) + + require.Len(t, group.Targets, 2) + + target2 := group.Targets[1] + require.Contains(t, target2, model.LabelName("__meta_custom_xds_label")) + require.Equal(t, model.LabelValue("a-value"), target2["__meta_custom_xds_label"]) +} + +func TestPollingDisappearingTargets(t *testing.T) { + server := "http://198.161.2.0" + source := "test" + rc := &testResourceClient{ + server: server, + protocolVersion: ProtocolV3, + fetch: func(ctx context.Context) (*v3.DiscoveryResponse, error) { + return &v3.DiscoveryResponse{}, nil + }, } - group2 := groups[1] - require.Contains(t, group2.Labels, model.LabelName("__meta_custom_xds_label")) - require.Equal(t, model.LabelValue("a-value"), group2.Labels["__meta_custom_xds_label"]) + // On the first poll, send back two targets. On the next, send just one. + counter := 0 + parser := func(resources []*anypb.Any, typeUrl string) ([]model.LabelSet, error) { + counter++ + if counter == 1 { + return []model.LabelSet{ + { + "__meta_custom_xds_label": "a-value", + "__address__": "10.1.4.32:9090", + "instance": "prometheus-01", + }, + { + "__meta_custom_xds_label": "a-value", + "__address__": "10.1.5.32:9090", + "instance": "prometheus-02", + }, + }, nil + } + + return []model.LabelSet{ + { + "__meta_custom_xds_label": "a-value", + "__address__": "10.1.4.32:9090", + "instance": "prometheus-01", + }, + }, nil + } + + pd := &fetchDiscovery{ + source: source, + client: rc, + logger: nopLogger, + fetchDuration: testFetchDuration, + fetchFailuresCount: testFetchFailuresCount, + fetchSkipUpdateCount: testFetchSkipUpdateCount, + parseResources: parser, + } + + ch := make(chan []*targetgroup.Group, 1) + pd.poll(context.Background(), ch) + groups := <-ch + require.NotNil(t, groups) + + require.Len(t, groups, 1) + + require.Equal(t, source, groups[0].Source) + require.Len(t, groups[0].Targets, 2) + + pd.poll(context.Background(), ch) + groups = <-ch + require.NotNil(t, groups) + + require.Len(t, groups, 1) + + require.Equal(t, source, groups[0].Source) + require.Len(t, groups[0].Targets, 1) } From 77d3f3722eec2776e2bd2151b6400e9ca3378885 Mon Sep 17 00:00:00 2001 From: Frederic Branczyk Date: Sat, 7 Aug 2021 17:37:48 +0200 Subject: [PATCH 14/19] *: cut v2.29.0-rc.2 Signed-off-by: Frederic Branczyk --- CHANGELOG.md | 2 +- VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 65c9dba40e..9e6dec38ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## 2.29.0-rc.1 / 2021-08-04 +## 2.29.0-rc.2 / 2021-08-09 Note for macOS users: Due to [changes in the upcoming Go 1.17](https://tip.golang.org/doc/go1.17#darwin), this is the last Prometheus release that supports macOS 10.12 Sierra. diff --git a/VERSION b/VERSION index 4239816eed..21070066cf 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.29.0-rc.1 +2.29.0-rc.2 From 0d7eb73d92278052b78d6bfb4fe480cbead3d48f Mon Sep 17 00:00:00 2001 From: Levi Harrison Date: Fri, 6 Aug 2021 21:54:44 -0600 Subject: [PATCH 15/19] Rename links Signed-off-by: Levi Harrison --- docs/querying/basics.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/querying/basics.md b/docs/querying/basics.md index 8c66c53624..f4c176ed7d 100644 --- a/docs/querying/basics.md +++ b/docs/querying/basics.md @@ -210,7 +210,7 @@ can be specified: rate(http_requests_total[5m] offset -1w) This feature is enabled by setting `--enable-feature=promql-negative-offset` -flag. See [disabled features](../disabled_features.md) for more details about +flag. See [feature flags](../feature_flags.md) for more details about this flag. ### @ modifier @@ -251,7 +251,7 @@ These 2 queries will produce the same result. This modifier is disabled by default since it breaks the invariant that PromQL does not look ahead of the evaluation time for samples. It can be enabled by setting -`--enable-feature=promql-at-modifier` flag. See [disabled features](../disabled_features.md) for more details about this flag. +`--enable-feature=promql-at-modifier` flag. See [feature flags](../feature_flags.md) for more details about this flag. Additionally, `start()` and `end()` can also be used as values for the `@` modifier as special values. From 765118e470c20122a45c2bc29bdcd02a11c00cda Mon Sep 17 00:00:00 2001 From: Augustin Husson Date: Tue, 10 Aug 2021 11:47:49 +0200 Subject: [PATCH 16/19] bump codemirror-promql to 0.17.0 Signed-off-by: Augustin Husson --- web/ui/react-app/package.json | 2 +- web/ui/react-app/yarn.lock | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/web/ui/react-app/package.json b/web/ui/react-app/package.json index 9bca4ab169..422efb4c76 100644 --- a/web/ui/react-app/package.json +++ b/web/ui/react-app/package.json @@ -22,7 +22,7 @@ "@nexucis/fuzzy": "^0.2.2", "@reach/router": "^1.2.1", "bootstrap": "^4.6.0", - "codemirror-promql": "^0.16.0", + "codemirror-promql": "^0.17.0", "css.escape": "^1.5.1", "downshift": "^3.4.8", "enzyme-to-json": "^3.4.3", diff --git a/web/ui/react-app/yarn.lock b/web/ui/react-app/yarn.lock index 2fba496ffd..5cfc3736a5 100644 --- a/web/ui/react-app/yarn.lock +++ b/web/ui/react-app/yarn.lock @@ -3479,12 +3479,12 @@ coa@^2.0.2: chalk "^2.4.1" q "^1.1.2" -codemirror-promql@^0.16.0: - version "0.16.0" - resolved "https://registry.yarnpkg.com/codemirror-promql/-/codemirror-promql-0.16.0.tgz#c51ca8ce1772a228ebfdbc8431550b2385670d46" - integrity sha512-/npytsj103ccWSMWffxibDuS+0p8DFneB7eRdUQULNVOsF+QSif2fqohjz6C9OmM+r58hA5qmdBUdhtogqsvEQ== +codemirror-promql@^0.17.0: + version "0.17.0" + resolved "https://registry.yarnpkg.com/codemirror-promql/-/codemirror-promql-0.17.0.tgz#ece2e6323040907993812bfa3bb00402b17ff8e7" + integrity sha512-etlSQ8t9FvNfnNXZSGDOsLDFIgFaUaYUWQVPEs/YLdG41VMFdhnFoC4zb4iHyx1DJjxBW/dHk7ZHLJDa9aNNzg== dependencies: - lezer-promql "^0.19.0" + lezer-promql "^0.20.0" lru-cache "^6.0.0" collection-visit@^1.0.0: @@ -7488,10 +7488,10 @@ levn@^0.3.0, levn@~0.3.0: prelude-ls "~1.1.2" type-check "~0.3.2" -lezer-promql@^0.19.0: - version "0.19.0" - resolved "https://registry.yarnpkg.com/lezer-promql/-/lezer-promql-0.19.0.tgz#0cedaead2d7b3700a25c09e5ae85568979806710" - integrity sha512-divgYjuKw4ESDWVCXg7FipH2dF4vq0aWTb0QCyIGz5NHTLx6H+tVC7IlMkQSqsK8t/6qhgxh6A9s6XrE4ZFFJQ== +lezer-promql@^0.20.0: + version "0.20.0" + resolved "https://registry.yarnpkg.com/lezer-promql/-/lezer-promql-0.20.0.tgz#d5d233fa5dfc5fb7fcd3efe0022fd31dd2f2d539" + integrity sha512-1CHG77fFghl032FfHT33buGyAHiTaMy2fqicEhcp2wWnbxZxS+Jt6gMzEUaf/TmRTIUJofj9uLar7iL22Jazug== lezer-tree@^0.13.0, lezer-tree@^0.13.2: version "0.13.2" From 92981dfe7e09a95cb218d3af0ae27d20e6174a2c Mon Sep 17 00:00:00 2001 From: Frederic Branczyk Date: Tue, 10 Aug 2021 21:31:44 +0200 Subject: [PATCH 17/19] *: cut v2.29.0 Signed-off-by: Frederic Branczyk --- CHANGELOG.md | 2 +- VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e6dec38ee..c6a4d69bc1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## 2.29.0-rc.2 / 2021-08-09 +## 2.29.0 / 2021-08-11 Note for macOS users: Due to [changes in the upcoming Go 1.17](https://tip.golang.org/doc/go1.17#darwin), this is the last Prometheus release that supports macOS 10.12 Sierra. diff --git a/VERSION b/VERSION index 21070066cf..f01356823f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.29.0-rc.2 +2.29.0 From 2d2aba4e8dde96d8a6e94d02042b9b3bf122f5c6 Mon Sep 17 00:00:00 2001 From: Julien Pivotto Date: Wed, 11 Aug 2021 16:11:49 +0200 Subject: [PATCH 18/19] tsdb: align atomically accessed int64 (#9192) This prevents a panic in 32-bit archs: https://pkg.go.dev/sync/atomic#pkg-note-BUG Fixed #9190 Signed-off-by: Julien Pivotto --- tsdb/head.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tsdb/head.go b/tsdb/head.go index ae549d2537..b0124922ed 100644 --- a/tsdb/head.go +++ b/tsdb/head.go @@ -117,6 +117,10 @@ type Head struct { // HeadOptions are parameters for the Head block. type HeadOptions struct { + // Runtime reloadable option. At the top of the struct for 32 bit OS: + // https://pkg.go.dev/sync/atomic#pkg-note-BUG + MaxExemplars atomic.Int64 + ChunkRange int64 // ChunkDirRoot is the parent directory of the chunks directory. ChunkDirRoot string @@ -128,9 +132,6 @@ type HeadOptions struct { StripeSize int SeriesCallback SeriesLifecycleCallback EnableExemplarStorage bool - - // Runtime reloadable options. - MaxExemplars atomic.Int64 } func DefaultHeadOptions() *HeadOptions { From dcb07e8eac34b5ea37cd229545000b857f1c1637 Mon Sep 17 00:00:00 2001 From: Julien Pivotto Date: Wed, 11 Aug 2021 16:12:03 +0200 Subject: [PATCH 19/19] Release 2.29.1 (#9193) Signed-off-by: Julien Pivotto --- CHANGELOG.md | 5 +++++ VERSION | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c6a4d69bc1..f96ebd4aca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## 2.29.1 / 2021-08-11 + +* [BUGFIX] tsdb: align atomically accessed int64 to prevent panic in 32-bit + archs. #9192 + ## 2.29.0 / 2021-08-11 Note for macOS users: Due to [changes in the upcoming Go 1.17](https://tip.golang.org/doc/go1.17#darwin), diff --git a/VERSION b/VERSION index f01356823f..69886179f9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.29.0 +2.29.1