diff --git a/.github/workflows/buf-lint.yml b/.github/workflows/buf-lint.yml index 31e4d7ee2a..d7adc93cf6 100644 --- a/.github/workflows/buf-lint.yml +++ b/.github/workflows/buf-lint.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: bufbuild/buf-setup-action@1115d0acd3d2a120b30023fac52abc46807c8fd6 # v1.48.0 + - uses: bufbuild/buf-setup-action@a47c93e0b1648d5651a065437926377d060baa99 # v1.50.0 with: github_token: ${{ secrets.GITHUB_TOKEN }} - uses: bufbuild/buf-lint-action@06f9dd823d873146471cfaaf108a993fe00e5325 # v1.1.1 diff --git a/.github/workflows/buf.yml b/.github/workflows/buf.yml index b72b927bb0..3384b4aa2f 100644 --- a/.github/workflows/buf.yml +++ b/.github/workflows/buf.yml @@ -13,7 +13,7 @@ jobs: if: github.repository_owner == 'prometheus' steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: bufbuild/buf-setup-action@1115d0acd3d2a120b30023fac52abc46807c8fd6 # v1.48.0 + - uses: bufbuild/buf-setup-action@a47c93e0b1648d5651a065437926377d060baa99 # v1.50.0 with: github_token: ${{ secrets.GITHUB_TOKEN }} - uses: bufbuild/buf-lint-action@06f9dd823d873146471cfaaf108a993fe00e5325 # v1.1.1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3bf12b07c4..57c47720c6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -80,7 +80,7 @@ jobs: runs-on: windows-latest steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0 + - uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0 with: go-version: 1.23.x - run: | @@ -171,7 +171,7 @@ jobs: - name: Checkout repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Go - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0 + uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0 with: cache: false go-version: 1.23.x @@ -184,14 +184,14 @@ jobs: - name: Checkout repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Go - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0 + uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0 with: go-version: 1.23.x - name: Install snmp_exporter/generator dependencies run: sudo apt-get update && sudo apt-get -y install libsnmp-dev if: github.repository == 'prometheus/snmp_exporter' - name: Lint - uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # v6.1.1 + uses: golangci/golangci-lint-action@ec5d18412c0aeab7936cb16880d708ba2a64e1ae # v6.2.0 with: args: --verbose # Make sure to sync this with Makefile.common and scripts/golangci-lint.yml. diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 9a522fb6c9..7e2a0af270 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -27,12 +27,12 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Initialize CodeQL - uses: github/codeql-action/init@48ab28a6f5dbc2a99bf1e0131198dd8f1df78169 # v3.28.0 + uses: github/codeql-action/init@dd746615b3b9d728a6a37ca2045b68ca76d4841a # v3.28.8 with: languages: ${{ matrix.language }} - name: Autobuild - uses: github/codeql-action/autobuild@48ab28a6f5dbc2a99bf1e0131198dd8f1df78169 # v3.28.0 + uses: github/codeql-action/autobuild@dd746615b3b9d728a6a37ca2045b68ca76d4841a # v3.28.8 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@48ab28a6f5dbc2a99bf1e0131198dd8f1df78169 # v3.28.0 + uses: github/codeql-action/analyze@dd746615b3b9d728a6a37ca2045b68ca76d4841a # v3.28.8 diff --git a/.github/workflows/fuzzing.yml b/.github/workflows/fuzzing.yml index 9f5ef96296..cc3b57d85d 100644 --- a/.github/workflows/fuzzing.yml +++ b/.github/workflows/fuzzing.yml @@ -21,7 +21,7 @@ jobs: fuzz-seconds: 600 dry-run: false - name: Upload Crash - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 if: failure() && steps.build.outcome == 'success' with: name: artifacts diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index 1415fead8e..85d467a37b 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -37,7 +37,7 @@ jobs: # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: "Upload artifact" - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # tag=v4.5.0 + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # tag=v4.6.0 with: name: SARIF file path: results.sarif @@ -45,6 +45,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@48ab28a6f5dbc2a99bf1e0131198dd8f1df78169 # tag=v3.28.0 + uses: github/codeql-action/upload-sarif@dd746615b3b9d728a6a37ca2045b68ca76d4841a # tag=v3.28.8 with: sarif_file: results.sarif diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index d71bcbc9d8..371d92a69a 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -11,7 +11,7 @@ jobs: if: github.repository_owner == 'prometheus' || github.repository_owner == 'prometheus-community' # Don't run this workflow on forks. runs-on: ubuntu-latest steps: - - uses: actions/stale@28ca1036281a5e5922ead5184a1bbf96e5fc984e # v9.0.0 + - uses: actions/stale@5bef64f19d7facfb25b37b414482c7164d639639 # v9.1.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} # opt out of defaults to avoid marking issues as stale and closing them diff --git a/docs/querying/basics.md b/docs/querying/basics.md index c1c76e7e04..8fc86bc230 100644 --- a/docs/querying/basics.md +++ b/docs/querying/basics.md @@ -47,7 +47,19 @@ _Notes about the experimental native histograms:_ disabling the feature flag again), both instant vectors and range vectors may now contain samples that aren't simple floating point numbers (float samples) but complete histograms (histogram samples). A vector may contain a mix of - float samples and histogram samples. + float samples and histogram samples. Note that the term “histogram sample” in + the PromQL documentation always refers to a native histogram. Classic + histograms are broken up into a number of series of float samples. From the + perspective of PromQL, there are no “classic histogram samples”. +* Like float samples, histogram samples can be counters or gauges, also called + counter histograms or gauge histograms, respectively. +* Native histograms can have different bucket layouts, but they are generally + convertible to compatible versions to apply binary and aggregation operations + to them. This is not true for all bucketing schemas. If incompatible + histograms are encountered in an operation, the corresponding output vector + element is removed from the result, flagged with a warn-level annotation. + More details can be found in the + [native histogram specification](https://prometheus.io/docs/specs/native_histograms/#compatibility-between-histograms). ## Literals diff --git a/docs/querying/functions.md b/docs/querying/functions.md index facdfce154..6615c2aebc 100644 --- a/docs/querying/functions.md +++ b/docs/querying/functions.md @@ -11,30 +11,17 @@ instant-vector)`. This means that there is one argument `v` which is an instant vector, which if not provided it will default to the value of the expression `vector(time())`. -_Notes about the experimental native histograms:_ - -* Ingesting native histograms has to be enabled via a [feature - flag](../feature_flags.md#native-histograms). As long as no native histograms - have been ingested into the TSDB, all functions will behave as usual. -* Functions that do not explicitly mention native histograms in their - documentation (see below) will ignore histogram samples. -* Functions that do already act on native histograms might still change their - behavior in the future. -* If a function requires the same bucket layout between multiple native - histograms it acts on, it will automatically convert them - appropriately. (With the currently supported bucket schemas, that's always - possible.) - ## `abs()` -`abs(v instant-vector)` returns the input vector with all sample values converted to -their absolute value. +`abs(v instant-vector)` returns a vector containing all float samples in the +input vector converted to their absolute value. Histogram samples in the input +vector are ignored silently. ## `absent()` `absent(v instant-vector)` returns an empty vector if the vector passed to it -has any elements (floats or native histograms) and a 1-element vector with the -value 1 if the vector passed to it has no elements. +has any elements (float samples or histogram samples) and a 1-element vector +with the value 1 if the vector passed to it has no elements. This is useful for alerting on when no time series exist for a given metric name and label combination. @@ -56,8 +43,9 @@ of the 1-element output vector from the input vector. ## `absent_over_time()` `absent_over_time(v range-vector)` returns an empty vector if the range vector -passed to it has any elements (floats or native histograms) and a 1-element -vector with the value 1 if the range vector passed to it has no elements. +passed to it has any elements (float samples or histogram samples) and a +1-element vector with the value 1 if the range vector passed to it has no +elements. This is useful for alerting on when no time series exist for a given metric name and label combination for a certain amount of time. @@ -78,8 +66,9 @@ labels of the 1-element output vector from the input vector. ## `ceil()` -`ceil(v instant-vector)` rounds the sample values of all elements in `v` up to -the nearest integer value greater than or equal to v. +`ceil(v instant-vector)` returns a vector containing all float samples in the +input vector rounded up to the nearest integer value greater than or equal to +their original value. Histogram samples in the input vector are ignored silently. * `ceil(+Inf) = +Inf` * `ceil(±0) = ±0` @@ -90,49 +79,62 @@ the nearest integer value greater than or equal to v. For each input time series, `changes(v range-vector)` returns the number of times its value has changed within the provided time range as an instant -vector. +vector. A float sample followed by a histogram sample, or vice versa, counts as +a change. A counter histogram sample followed by a gauge histogram sample with +otherwise exactly the same values, or vice versa, does not count as a change. ## `clamp()` -`clamp(v instant-vector, min scalar, max scalar)` -clamps the sample values of all elements in `v` to have a lower limit of `min` and an upper limit of `max`. +`clamp(v instant-vector, min scalar, max scalar)` clamps the values of all +float samples in `v` to have a lower limit of `min` and an upper limit of +`max`. Histogram samples in the input vector are ignored silently. Special cases: * Return an empty vector if `min > max` -* Return `NaN` if `min` or `max` is `NaN` +* Float samples are clamped to `NaN` if `min` or `max` is `NaN` ## `clamp_max()` -`clamp_max(v instant-vector, max scalar)` clamps the sample values of all -elements in `v` to have an upper limit of `max`. +`clamp_max(v instant-vector, max scalar)` clamps the values of all float +samples in `v` to have an upper limit of `max`. Histogram samples in the input +vector are ignored silently. ## `clamp_min()` -`clamp_min(v instant-vector, min scalar)` clamps the sample values of all -elements in `v` to have a lower limit of `min`. +`clamp_min(v instant-vector, min scalar)` clamps the values of all float +samples in `v` to have a lower limit of `min`. Histogram samples in the input +vector are ignored silently. ## `day_of_month()` -`day_of_month(v=vector(time()) instant-vector)` returns the day of the month -for each of the given times in UTC. Returned values are from 1 to 31. +`day_of_month(v=vector(time()) instant-vector)` interpretes float samples in +`v` as timestamps (number of seconds since January 1, 1970 UTC) and returns the +day of the month (in UTC) for each of those timestamps. Returned values are +from 1 to 31. Histogram samples in the input vector are ignored silently. ## `day_of_week()` -`day_of_week(v=vector(time()) instant-vector)` returns the day of the week for -each of the given times in UTC. Returned values are from 0 to 6, where 0 means -Sunday etc. +`day_of_week(v=vector(time()) instant-vector)` interpretes float samples in `v` +as timestamps (number of seconds since January 1, 1970 UTC) and returns the day +of the week (in UTC) for each of those timestamps. Returned values are from 0 +to 6, where 0 means Sunday etc. Histogram samples in the input vector are +ignored silently. -## `day_of_year()` +## `day_of_year()` -`day_of_year(v=vector(time()) instant-vector)` returns the day of the year for -each of the given times in UTC. Returned values are from 1 to 365 for non-leap years, -and 1 to 366 in leap years. +`day_of_year(v=vector(time()) instant-vector)` interpretes float samples in `v` +as timestamps (number of seconds since January 1, 1970 UTC) and returns the day +of the year (in UTC) for each of those timestamps. Returned values are from 1 +to 365 for non-leap years, and 1 to 366 in leap years. Histogram samples in the +input vector are ignored silently. ## `days_in_month()` -`days_in_month(v=vector(time()) instant-vector)` returns number of days in the -month for each of the given times in UTC. Returned values are from 28 to 31. +`days_in_month(v=vector(time()) instant-vector)` interpretes float samples in +`v` as timestamps (number of seconds since January 1, 1970 UTC) and returns the +number of days in the month of each of those timestamps (in UTC). Returned +values are from 28 to 31. Histogram samples in the input vector are ignored silently. ## `delta()` @@ -150,36 +152,67 @@ between now and 2 hours ago: delta(cpu_temp_celsius{host="zeus"}[2h]) ``` -`delta` acts on native histograms by calculating a new histogram where each +`delta` acts on histogram samples by calculating a new histogram where each component (sum and count of observations, buckets) is the difference between -the respective component in the first and last native histogram in -`v`. However, each element in `v` that contains a mix of float and native -histogram samples within the range, will be missing from the result vector. +the respective component in the first and last native histogram in `v`. +However, each element in `v` that contains a mix of float samples and histogram +samples within the range will be omitted from the result vector, flagged by a +warn-level annotation. -`delta` should only be used with gauges and native histograms where the -components behave like gauges (so-called gauge histograms). +`delta` should only be used with gauges (for both floats and histograms). ## `deriv()` -`deriv(v range-vector)` calculates the per-second derivative of the time series in a range -vector `v`, using [simple linear regression](https://en.wikipedia.org/wiki/Simple_linear_regression). -The range vector must have at least two samples in order to perform the calculation. When `+Inf` or -`-Inf` are found in the range vector, the slope and offset value calculated will be `NaN`. +`deriv(v range-vector)` calculates the per-second derivative of each float time +series in the range vector `v`, using [simple linear +regression](https://en.wikipedia.org/wiki/Simple_linear_regression). The range +vector must have at least two float samples in order to perform the +calculation. When `+Inf` or `-Inf` are found in the range vector, the slope and +offset value calculated will be `NaN`. -`deriv` should only be used with gauges. +`deriv` should only be used with gauges and only works for float samples. +Elements in the range vector that contain only histogram samples are ignored +entirely. For elements that contain a mix of float and histogram samples, only +the float samples are used as input, which is flagged by an info-level +annotation. + +## `double_exponential_smoothing()` + +**This function has to be enabled via the [feature +flag](../feature_flags.md#experimental-promql-functions) +`--enable-feature=promql-experimental-functions`.** + +`double_exponential_smoothing(v range-vector, sf scalar, tf scalar)` produces a +smoothed value for each float time series in the range in `v`. The lower the +smoothing factor `sf`, the more importance is given to old data. The higher the +trend factor `tf`, the more trends in the data is considered. Both `sf` and +`tf` must be between 0 and 1. For additional details, refer to [NIST +Engineering Statistics +Handbook](https://www.itl.nist.gov/div898/handbook/pmc/section4/pmc433.htm). In +Prometheus V2 this function was called `holt_winters`. This caused confusion +since the Holt-Winters method usually refers to triple exponential smoothing. +Double exponential smoothing as implemented here is also referred to as "Holt +Linear". + +`double_exponential_smoothing` should only be used with gauges and only works +for float samples. Elements in the range vector that contain only histogram +samples are ignored entirely. For elements that contain a mix of float and +histogram samples, only the float samples are used as input, which is flagged +by an info-level annotation. ## `exp()` -`exp(v instant-vector)` calculates the exponential function for all elements in `v`. -Special cases are: +`exp(v instant-vector)` calculates the exponential function for all float +samples in `v`. Histogram samples are ignored silently. Special cases are: * `Exp(+Inf) = +Inf` * `Exp(NaN) = NaN` ## `floor()` -`floor(v instant-vector)` rounds the sample values of all elements in `v` down -to the nearest integer value smaller than or equal to v. +`floor(v instant-vector)` returns a vector containing all float samples in the +input vector rounded down to the nearest integer value smaller than or equal +to their original value. Histogram samples in the input vector are ignored silently. * `floor(+Inf) = +Inf` * `floor(±0) = ±0` @@ -188,12 +221,8 @@ to the nearest integer value smaller than or equal to v. ## `histogram_avg()` -_This function only acts on native histograms, which are an experimental -feature. The behavior of this function may change in future versions of -Prometheus, including its removal from PromQL._ - -`histogram_avg(v instant-vector)` returns the arithmetic average of observed values stored in -a native histogram. Samples that are not native histograms are ignored and do +`histogram_avg(v instant-vector)` returns the arithmetic average of observed +values stored in each histogram sample in `v`. Float samples are ignored and do not show up in the returned vector. Use `histogram_avg` as demonstrated below to compute the average request duration @@ -209,32 +238,25 @@ Which is equivalent to the following query: ## `histogram_count()` and `histogram_sum()` -_Both functions only act on native histograms, which are an experimental -feature. The behavior of these functions may change in future versions of -Prometheus, including their removal from PromQL._ - `histogram_count(v instant-vector)` returns the count of observations stored in -a native histogram. Samples that are not native histograms are ignored and do -not show up in the returned vector. +each histogram sample in `v`. Float samples are ignored and do not show up in +the returned vector. Similarly, `histogram_sum(v instant-vector)` returns the sum of observations -stored in a native histogram. +stored in each histogram sample. Use `histogram_count` in the following way to calculate a rate of observations -(in this case corresponding to “requests per second”) from a native histogram: +(in this case corresponding to “requests per second”) from a series of +histogram samples: histogram_count(rate(http_request_duration_seconds[10m])) ## `histogram_fraction()` -_This function only acts on native histograms, which are an experimental -feature. The behavior of this function may change in future versions of -Prometheus, including its removal from PromQL._ - -For a native histogram, `histogram_fraction(lower scalar, upper scalar, v -instant-vector)` returns the estimated fraction of observations between the -provided lower and upper values. Samples that are not native histograms are -ignored and do not show up in the returned vector. +`histogram_fraction(lower scalar, upper scalar, v instant-vector)` returns the +estimated fraction of observations between the provided lower and upper values +for each histogram sample in `v`. Float samples are ignored and do not show up +in the returned vector. For example, the following expression calculates the fraction of HTTP requests over the last hour that took 200ms or less: @@ -253,12 +275,13 @@ observations less than or equal 0.2 would be `-Inf` rather than `0`. Whether the provided boundaries are inclusive or exclusive is only relevant if the provided boundaries are precisely aligned with bucket boundaries in the underlying native histogram. In this case, the behavior depends on the schema -definition of the histogram. The currently supported schemas all feature -inclusive upper boundaries and exclusive lower boundaries for positive values -(and vice versa for negative values). Without a precise alignment of -boundaries, the function uses linear interpolation to estimate the -fraction. With the resulting uncertainty, it becomes irrelevant if the -boundaries are inclusive or exclusive. +definition of the histogram. (The usual standard exponential schemas all +feature inclusive upper boundaries and exclusive lower boundaries for positive +values, and vice versa for negative values.) Without a precise alignment of +boundaries, the function uses interpolation to estimate the fraction. With the +resulting uncertainty, it becomes irrelevant if the boundaries are inclusive or +exclusive. The interpolation method is the same as the one used for +`histogram_quantile()`. See there for more details. ## `histogram_quantile()` @@ -270,10 +293,6 @@ summaries](https://prometheus.io/docs/practices/histograms) for a detailed explanation of φ-quantiles and the usage of the (classic) histogram metric type in general.) -_Note that native histograms are an experimental feature. The behavior of this -function when dealing with native histograms may change in future versions of -Prometheus._ - The float samples in `b` are considered the counts of observations in each bucket of one or more classic histograms. Each float sample must have a label `le` where the label value denotes the inclusive upper bound of the bucket. @@ -284,8 +303,8 @@ type](https://prometheus.io/docs/concepts/metric_types/#histogram) automatically provides time series with the `_bucket` suffix and the appropriate labels. -The native histogram samples in `b` are treated each individually as a separate -histogram to calculate the quantile from. +The (native) histogram samples in `b` are treated each individually as a +separate histogram to calculate the quantile from. As long as no naming collisions arise, `b` may contain a mix of classic and native histograms. @@ -336,7 +355,9 @@ non-zero-buckets of native histograms with a standard exponential bucketing schema, the interpolation is done under the assumption that the samples within the bucket are distributed in a way that they would uniformly populate the buckets in a hypothetical histogram with higher resolution. (This is also -called _exponential interpolation_.) +called _exponential interpolation_. See the [native histogram +specification](https://prometheus.io/docs/specs/native_histograms/#interpolation-within-a-bucket) +for more details.) If `b` has 0 observations, `NaN` is returned. For φ < 0, `-Inf` is returned. For φ > 1, `+Inf` is returned. For φ = `NaN`, `NaN` is returned. @@ -387,63 +408,46 @@ difference between two buckets is a trillionth (1e-12) of the sum of both buckets.) Furthermore, if there are non-monotonic bucket counts even after this adjustment, they are increased to the value of the previous buckets to enforce monotonicity. The latter is evidence for an actual issue with the input data -and is therefore flagged with an informational annotation reading `input to +and is therefore flagged by an info-level annotation reading `input to histogram_quantile needed to be fixed for monotonicity`. If you encounter this annotation, you should find and remove the source of the invalid data. ## `histogram_stddev()` and `histogram_stdvar()` -_Both functions only act on native histograms, which are an experimental -feature. The behavior of these functions may change in future versions of -Prometheus, including their removal from PromQL._ - `histogram_stddev(v instant-vector)` returns the estimated standard deviation -of observations in a native histogram, based on the geometric mean of the buckets -where the observations lie. Samples that are not native histograms are ignored and -do not show up in the returned vector. +of observations for each histogram sample in `v`, based on the geometric mean +of the buckets where the observations lie. Float samples are ignored and do not +show up in the returned vector. Similarly, `histogram_stdvar(v instant-vector)` returns the estimated standard -variance of observations in a native histogram. - -## `double_exponential_smoothing()` - -**This function has to be enabled via the [feature flag](../feature_flags.md#experimental-promql-functions) `--enable-feature=promql-experimental-functions`.** - -`double_exponential_smoothing(v range-vector, sf scalar, tf scalar)` produces a smoothed value -for time series based on the range in `v`. The lower the smoothing factor `sf`, -the more importance is given to old data. The higher the trend factor `tf`, the -more trends in the data is considered. Both `sf` and `tf` must be between 0 and -1. -For additional details, refer to [NIST Engineering Statistics Handbook](https://www.itl.nist.gov/div898/handbook/pmc/section4/pmc433.htm). -In Prometheus V2 this function was called `holt_winters`. This caused confusion -since the Holt-Winters method usually refers to triple exponential smoothing. -Double exponential smoothing as implemented here is also referred to as "Holt -Linear". - -`double_exponential_smoothing` should only be used with gauges. +variance of observations for each histogram sample in `v`. ## `hour()` -`hour(v=vector(time()) instant-vector)` returns the hour of the day -for each of the given times in UTC. Returned values are from 0 to 23. +`hour(v=vector(time()) instant-vector)` interpretes float samples in `v` as +timestamps (number of seconds since January 1, 1970 UTC) and returns the hour +of the day (in UTC) for each of those timestamps. Returned values are from 0 +to 23. Histogram samples in the input vector are ignored silently. ## `idelta()` `idelta(v range-vector)` calculates the difference between the last two samples in the range vector `v`, returning an instant vector with the given deltas and -equivalent labels. +equivalent labels. Both samples must be either float samples or histogram +samples. Elements in `v` where one of the last two samples is a float sample +and the other is a histogram sample will be omitted from the result vector, +flagged by a warn-level annotation. -`idelta` should only be used with gauges. +`idelta` should only be used with gauges (for both floats and histograms). ## `increase()` -`increase(v range-vector)` calculates the increase in the -time series in the range vector. Breaks in monotonicity (such as counter -resets due to target restarts) are automatically adjusted for. The -increase is extrapolated to cover the full time range as specified -in the range vector selector, so that it is possible to get a -non-integer result even if a counter increases only by integer -increments. +`increase(v range-vector)` calculates the increase in the time series in the +range vector. Breaks in monotonicity (such as counter resets due to target +restarts) are automatically adjusted for. The increase is extrapolated to cover +the full time range as specified in the range vector selector, so that it is +possible to get a non-integer result even if a counter increases only by +integer increments. The following example expression returns the number of HTTP requests as measured over the last 5 minutes, per time series in the range vector: @@ -452,19 +456,20 @@ over the last 5 minutes, per time series in the range vector: increase(http_requests_total{job="api-server"}[5m]) ``` -`increase` acts on native histograms by calculating a new histogram where each -component (sum and count of observations, buckets) is the increase between -the respective component in the first and last native histogram in -`v`. However, each element in `v` that contains a mix of float and native -histogram samples within the range, will be missing from the result vector. +`increase` acts on histogram samples by calculating a new histogram where each +component (sum and count of observations, buckets) is the increase between the +respective component in the first and last native histogram in `v`. However, +each element in `v` that contains a mix of float samples and histogram samples +within the range, will be omitted from the result vector, flagged by a +warn-level annotation. -`increase` should only be used with counters and native histograms where the -components behave like counters. It is syntactic sugar for `rate(v)` multiplied -by the number of seconds under the specified time range window, and should be -used primarily for human readability. Use `rate` in recording rules so that -increases are tracked consistently on a per-second basis. +`increase` should only be used with counters (for both floats and histograms). +It is syntactic sugar for `rate(v)` multiplied by the number of seconds under +the specified time range window, and should be used primarily for human +readability. Use `rate` in recording rules so that increases are tracked +consistently on a per-second basis. -## `info()` (experimental) +## `info()` _The `info` function is an experiment to improve UX around including labels from [info metrics](https://grafana.com/blog/2021/08/04/how-to-use-promql-joins-for-more-effective-queries-of-prometheus-metrics-at-scale/#info-metrics). @@ -560,7 +565,12 @@ consider all matching info series and with their appropriate identifying labels. `irate(v range-vector)` calculates the per-second instant rate of increase of the time series in the range vector. This is based on the last two data points. Breaks in monotonicity (such as counter resets due to target restarts) are -automatically adjusted for. +automatically adjusted for. Both samples must be either float samples or +histogram samples. Elements in `v` where one of the last two samples is a float +sample and the other is a histogram sample will be omitted from the result +vector, flagged by a warn-level annotation. + +`irate` should only be used with counters (for both floats and histograms). The following example expression returns the per-second rate of HTTP requests looking up to 5 minutes back for the two most recent data points, per time @@ -618,8 +628,8 @@ label_replace(up{job="api-server",service="a:c"}, "foo", "$name", "service", "(? ## `ln()` -`ln(v instant-vector)` calculates the natural logarithm for all elements in `v`. -Special cases are: +`ln(v instant-vector)` calculates the natural logarithm for all float samples +in `v`. Histogram samples in the input vector are ignored silently. Special cases are: * `ln(+Inf) = +Inf` * `ln(0) = -Inf` @@ -628,35 +638,45 @@ Special cases are: ## `log2()` -`log2(v instant-vector)` calculates the binary logarithm for all elements in `v`. -The special cases are equivalent to those in `ln`. +`log2(v instant-vector)` calculates the binary logarithm for all float samples +in `v`. Histogram samples in the input vector are ignored silently. The special cases +are equivalent to those in `ln`. ## `log10()` -`log10(v instant-vector)` calculates the decimal logarithm for all elements in `v`. -The special cases are equivalent to those in `ln`. +`log10(v instant-vector)` calculates the decimal logarithm for all float +samples in `v`. Histogram samples in the input vector are ignored silently. The special +cases are equivalent to those in `ln`. ## `minute()` -`minute(v=vector(time()) instant-vector)` returns the minute of the hour for each -of the given times in UTC. Returned values are from 0 to 59. +`minute(v=vector(time()) instant-vector)` interpretes float samples in `v` as +timestamps (number of seconds since January 1, 1970 UTC) and returns the minute +of the hour (in UTC) for each of those timestamps. Returned values are from 0 +to 59. Histogram samples in the input vector are ignored silently. ## `month()` -`month(v=vector(time()) instant-vector)` returns the month of the year for each -of the given times in UTC. Returned values are from 1 to 12, where 1 means -January etc. +`month(v=vector(time()) instant-vector)` interpretes float samples in `v` as +timestamps (number of seconds since January 1, 1970 UTC) and returns the month +of the year (in UTC) for each of those timestamps. Returned values are from 1 +to 12, where 1 means January etc. Histogram samples in the input vector are +ignored silently. ## `predict_linear()` `predict_linear(v range-vector, t scalar)` predicts the value of time series `t` seconds from now, based on the range vector `v`, using [simple linear -regression](https://en.wikipedia.org/wiki/Simple_linear_regression). -The range vector must have at least two samples in order to perform the -calculation. When `+Inf` or `-Inf` are found in the range vector, -the slope and offset value calculated will be `NaN`. +regression](https://en.wikipedia.org/wiki/Simple_linear_regression). The range +vector must have at least two float samples in order to perform the +calculation. When `+Inf` or `-Inf` are found in the range vector, the predicted +value will be `NaN`. -`predict_linear` should only be used with gauges. +`predict_linear` should only be used with gauges and only works for float +samples. Elements in the range vector that contain only histogram samples are +ignored entirely. For elements that contain a mix of float and histogram +samples, only the float samples are used as input, which is flagged by an +info-level annotation. ## `rate()` @@ -666,7 +686,7 @@ resets due to target restarts) are automatically adjusted for. Also, the calculation extrapolates to the ends of the time range, allowing for missed scrapes or imperfect alignment of scrape cycles with the range's time period. -The following example expression returns the per-second rate of HTTP requests as measured +The following example expression returns the per-second average rate of HTTP requests over the last 5 minutes, per time series in the range vector: ``` @@ -675,13 +695,13 @@ rate(http_requests_total{job="api-server"}[5m]) `rate` acts on native histograms by calculating a new histogram where each component (sum and count of observations, buckets) is the rate of increase -between the respective component in the first and last native histogram in -`v`. However, each element in `v` that contains a mix of float and native -histogram samples within the range, will be missing from the result vector. +between the respective component in the first and last native histogram in `v`. +However, each element in `v` that contains a mix of float and native histogram +samples within the range, will be omitted from the result vector, flagged by a +warn-level annotation. -`rate` should only be used with counters and native histograms where the -components behave like counters. It is best suited for alerting, and for -graphing of slow-moving counters. +`rate` should only be used with counters (for both floats and histograms). It +is best suited for alerting, and for graphing of slow-moving counters. Note that when combining `rate()` with an aggregation operator (e.g. `sum()`) or a function aggregating over time (any function ending in `_over_time`), @@ -696,17 +716,15 @@ decrease in the value between two consecutive float samples is interpreted as a counter reset. A reset in a native histogram is detected in a more complex way: Any decrease in any bucket, including the zero bucket, or in the count of observation constitutes a counter reset, but also the disappearance of any -previously populated bucket, an increase in bucket resolution, or a decrease of -the zero-bucket width. +previously populated bucket, a decrease of the zero-bucket width, or any schema +change that is not a compatible decrease of resolution. -`resets` should only be used with counters and counter-like native -histograms. +`resets` should only be used with counters (for both floats and histograms). -If the range vector contains a mix of float and histogram samples for the same -series, counter resets are detected separately and their numbers added up. The -change from a float to a histogram sample is _not_ considered a counter -reset. Each float sample is compared to the next float sample, and each -histogram is comprared to the next histogram. +A float sample followed by a histogram sample, or vice versa, counts as a +reset. A counter histogram sample followed by a gauge histogram sample, or vice +versa, also counts as a reset (but note that `resets` should not be used on +gauges in the first place, see above). ## `round()` @@ -714,53 +732,63 @@ histogram is comprared to the next histogram. elements in `v` to the nearest integer. Ties are resolved by rounding up. The optional `to_nearest` argument allows specifying the nearest multiple to which the sample values should be rounded. This multiple may also be a fraction. +Histogram samples in the input vector are ignored silently. ## `scalar()` -Given a single-element input vector, `scalar(v instant-vector)` returns the -sample value of that single element as a scalar. If the input vector does not -have exactly one element, `scalar` will return `NaN`. +Given an input vector that contains only one element with a float sample, +`scalar(v instant-vector)` returns the sample value of that float sample as a +scalar. If the input vector does not have exactly one element with a float +sample, `scalar` will return `NaN`. Histogram samples in the input vector are +ignored silently. ## `sgn()` -`sgn(v instant-vector)` returns a vector with all sample values converted to their sign, defined as this: 1 if v is positive, -1 if v is negative and 0 if v is equal to zero. +`sgn(v instant-vector)` returns a vector with all float sample values converted +to their sign, defined as this: 1 if v is positive, -1 if v is negative and 0 +if v is equal to zero. Histogram samples in the input vector are ignored silently. ## `sort()` -`sort(v instant-vector)` returns vector elements sorted by their sample values, -in ascending order. Native histograms are sorted by their sum of observations. +`sort(v instant-vector)` returns vector elements sorted by their float sample +values, in ascending order. Histogram samples in the input vector are ignored silently. -Please note that `sort` only affects the results of instant queries, as range query results always have a fixed output ordering. +Please note that `sort` only affects the results of instant queries, as range +query results always have a fixed output ordering. ## `sort_desc()` Same as `sort`, but sorts in descending order. -Like `sort`, `sort_desc` only affects the results of instant queries, as range query results always have a fixed output ordering. - ## `sort_by_label()` -**This function has to be enabled via the [feature flag](../feature_flags.md#experimental-promql-functions) `--enable-feature=promql-experimental-functions`.** +**This function has to be enabled via the [feature +flag](../feature_flags.md#experimental-promql-functions) +`--enable-feature=promql-experimental-functions`.** -`sort_by_label(v instant-vector, label string, ...)` returns vector elements sorted by the values of the given labels in ascending order. In case these label values are equal, elements are sorted by their full label sets. +`sort_by_label(v instant-vector, label string, ...)` returns vector elements +sorted by the values of the given labels in ascending order. In case these +label values are equal, elements are sorted by their full label sets. +`sort_by_label` acts on float and histogram samples in the same way. -Please note that the sort by label functions only affect the results of instant queries, as range query results always have a fixed output ordering. +Please note that `sort_by_label` only affect the results of instant queries, as +range query results always have a fixed output ordering. -This function uses [natural sort order](https://en.wikipedia.org/wiki/Natural_sort_order). +`sort_by_label` uses [natural sort +order](https://en.wikipedia.org/wiki/Natural_sort_order). ## `sort_by_label_desc()` -**This function has to be enabled via the [feature flag](../feature_flags.md#experimental-promql-functions) `--enable-feature=promql-experimental-functions`.** +**This function has to be enabled via the [feature +flag](../feature_flags.md#experimental-promql-functions) +`--enable-feature=promql-experimental-functions`.** Same as `sort_by_label`, but sorts in descending order. -Please note that the sort by label functions only affect the results of instant queries, as range query results always have a fixed output ordering. - -This function uses [natural sort order](https://en.wikipedia.org/wiki/Natural_sort_order). - ## `sqrt()` -`sqrt(v instant-vector)` calculates the square root of all elements in `v`. +`sqrt(v instant-vector)` calculates the square root of all float samples in +`v`. Histogram samples in the input vector are ignored silently. ## `time()` @@ -771,66 +799,80 @@ expression is to be evaluated. ## `timestamp()` `timestamp(v instant-vector)` returns the timestamp of each of the samples of -the given vector as the number of seconds since January 1, 1970 UTC. It also -works with histogram samples. +the given vector as the number of seconds since January 1, 1970 UTC. It acts on +float and histogram samples in the same way. ## `vector()` -`vector(s scalar)` returns the scalar `s` as a vector with no labels. +`vector(s scalar)` converts the scalar `s` to a float sample and returns it as +a single-element instant vector with no labels. ## `year()` -`year(v=vector(time()) instant-vector)` returns the year -for each of the given times in UTC. +`year(v=vector(time()) instant-vector)` returns the year for each of the given +times in UTC. Histogram samples in the input vector are ignored silently. ## `_over_time()` The following functions allow aggregating each series of a given range vector over time and return an instant vector with per-series aggregation results: -* `avg_over_time(range-vector)`: the average value of all points in the specified interval. -* `min_over_time(range-vector)`: the minimum value of all points in the specified interval. -* `max_over_time(range-vector)`: the maximum value of all points in the specified interval. -* `sum_over_time(range-vector)`: the sum of all values in the specified interval. -* `count_over_time(range-vector)`: the count of all values in the specified interval. -* `quantile_over_time(scalar, range-vector)`: the φ-quantile (0 ≤ φ ≤ 1) of the values in the specified interval. -* `stddev_over_time(range-vector)`: the population standard deviation of the values in the specified interval. -* `stdvar_over_time(range-vector)`: the population standard variance of the values in the specified interval. -* `last_over_time(range-vector)`: the most recent point value in the specified interval. +* `avg_over_time(range-vector)`: the average value of all float or histogram samples in the specified interval (see details below). +* `min_over_time(range-vector)`: the minimum value of all float samples in the specified interval. +* `max_over_time(range-vector)`: the maximum value of all float samples in the specified interval. +* `sum_over_time(range-vector)`: the sum of all float or histogram samples in the specified interval (see details below). +* `count_over_time(range-vector)`: the count of all samples in the specified interval. +* `quantile_over_time(scalar, range-vector)`: the φ-quantile (0 ≤ φ ≤ 1) of all float samples in the specified interval. +* `stddev_over_time(range-vector)`: the population standard deviation of all float samples in the specified interval. +* `stdvar_over_time(range-vector)`: the population standard variance of all float samples in the specified interval. +* `last_over_time(range-vector)`: the most recent sample in the specified interval. * `present_over_time(range-vector)`: the value 1 for any series in the specified interval. If the [feature flag](../feature_flags.md#experimental-promql-functions) `--enable-feature=promql-experimental-functions` is set, the following additional functions are available: -* `mad_over_time(range-vector)`: the median absolute deviation of all points in the specified interval. +* `mad_over_time(range-vector)`: the median absolute deviation of all float + samples in the specified interval. Note that all values in the specified interval have the same weight in the aggregation even if the values are not equally spaced throughout the interval. -`avg_over_time`, `sum_over_time`, `count_over_time`, `last_over_time`, and -`present_over_time` handle native histograms as expected. All other functions -ignore histogram samples. +These functions act on histograms in the following way: + +- `count_over_time`, `last_over_time`, and `present_over_time()` act on float + and histogram samples in the same way. +- `avg_over_time()` and `sum_over_time()` act on histogram samples in a way + that corresponds to the respective aggregation operators. If a series + contains a mix of float samples and histogram samples within the range, the + corresponding result is removed entirely from the output vector. Such a + removal is flagged by a warn-level annotation. +- All other functions ignore histogram samples in the following way: Input + ranges containing only histogram samples are silently removed from the + output. For ranges with a mix of histogram and float samples, only the float + samples are processed and the omission of the histogram samples is flagged by + an info-level annotation. ## Trigonometric Functions -The trigonometric functions work in radians: +The trigonometric functions work in radians. They ignore histogram samples in +the input vector. -* `acos(v instant-vector)`: calculates the arccosine of all elements in `v` ([special cases](https://pkg.go.dev/math#Acos)). -* `acosh(v instant-vector)`: calculates the inverse hyperbolic cosine of all elements in `v` ([special cases](https://pkg.go.dev/math#Acosh)). -* `asin(v instant-vector)`: calculates the arcsine of all elements in `v` ([special cases](https://pkg.go.dev/math#Asin)). -* `asinh(v instant-vector)`: calculates the inverse hyperbolic sine of all elements in `v` ([special cases](https://pkg.go.dev/math#Asinh)). -* `atan(v instant-vector)`: calculates the arctangent of all elements in `v` ([special cases](https://pkg.go.dev/math#Atan)). -* `atanh(v instant-vector)`: calculates the inverse hyperbolic tangent of all elements in `v` ([special cases](https://pkg.go.dev/math#Atanh)). -* `cos(v instant-vector)`: calculates the cosine of all elements in `v` ([special cases](https://pkg.go.dev/math#Cos)). -* `cosh(v instant-vector)`: calculates the hyperbolic cosine of all elements in `v` ([special cases](https://pkg.go.dev/math#Cosh)). -* `sin(v instant-vector)`: calculates the sine of all elements in `v` ([special cases](https://pkg.go.dev/math#Sin)). -* `sinh(v instant-vector)`: calculates the hyperbolic sine of all elements in `v` ([special cases](https://pkg.go.dev/math#Sinh)). -* `tan(v instant-vector)`: calculates the tangent of all elements in `v` ([special cases](https://pkg.go.dev/math#Tan)). -* `tanh(v instant-vector)`: calculates the hyperbolic tangent of all elements in `v` ([special cases](https://pkg.go.dev/math#Tanh)). +* `acos(v instant-vector)`: calculates the arccosine of all float samples in `v` ([special cases](https://pkg.go.dev/math#Acos)). +* `acosh(v instant-vector)`: calculates the inverse hyperbolic cosine of all float samples in `v` ([special cases](https://pkg.go.dev/math#Acosh)). +* `asin(v instant-vector)`: calculates the arcsine of all float samples in `v` ([special cases](https://pkg.go.dev/math#Asin)). +* `asinh(v instant-vector)`: calculates the inverse hyperbolic sine of all float samples in `v` ([special cases](https://pkg.go.dev/math#Asinh)). +* `atan(v instant-vector)`: calculates the arctangent of all float samples in `v` ([special cases](https://pkg.go.dev/math#Atan)). +* `atanh(v instant-vector)`: calculates the inverse hyperbolic tangent of all float samples in `v` ([special cases](https://pkg.go.dev/math#Atanh)). +* `cos(v instant-vector)`: calculates the cosine of all float samples in `v` ([special cases](https://pkg.go.dev/math#Cos)). +* `cosh(v instant-vector)`: calculates the hyperbolic cosine of all float samples in `v` ([special cases](https://pkg.go.dev/math#Cosh)). +* `sin(v instant-vector)`: calculates the sine of all float samples in `v` ([special cases](https://pkg.go.dev/math#Sin)). +* `sinh(v instant-vector)`: calculates the hyperbolic sine of all float samples in `v` ([special cases](https://pkg.go.dev/math#Sinh)). +* `tan(v instant-vector)`: calculates the tangent of all float samples in `v` ([special cases](https://pkg.go.dev/math#Tan)). +* `tanh(v instant-vector)`: calculates the hyperbolic tangent of all float samples in `v` ([special cases](https://pkg.go.dev/math#Tanh)). The following are useful for converting between degrees and radians: -* `deg(v instant-vector)`: converts radians to degrees for all elements in `v`. +* `deg(v instant-vector)`: converts radians to degrees for all float samples in `v`. * `pi()`: returns pi. -* `rad(v instant-vector)`: converts degrees to radians for all elements in `v`. +* `rad(v instant-vector)`: converts degrees to radians for all float samples in `v`. diff --git a/docs/querying/operators.md b/docs/querying/operators.md index f5f217ff63..02cea500a1 100644 --- a/docs/querying/operators.md +++ b/docs/querying/operators.md @@ -23,22 +23,51 @@ The following binary arithmetic operators exist in Prometheus: * `^` (power/exponentiation) Binary arithmetic operators are defined between scalar/scalar, vector/scalar, -and vector/vector value pairs. +and vector/vector value pairs. They follow the usual [IEEE 754 floating point +arithmetic](https://en.wikipedia.org/wiki/IEEE_754), including the handling of +special values like `NaN`, `+Inf`, and `-Inf`. **Between two scalars**, the behavior is obvious: they evaluate to another scalar that is the result of the operator applied to both scalar operands. **Between an instant vector and a scalar**, the operator is applied to the -value of every data sample in the vector. E.g. if a time series instant vector -is multiplied by 2, the result is another vector in which every sample value of -the original vector is multiplied by 2. The metric name is dropped. +value of every data sample in the vector. If the data sample is a float, the +operation performed on the data sample is again obvious, e.g. if an instant +vector of float samples is multiplied by 2, the result is another vector of +float samples in which every sample value of the original vector is multiplied +by 2. For vector elements that are histogram samples, the behavior is the +following: For `*`, all bucket populations and the count and the sum of +observations are multiplied by the scalar. For `/`, the histogram sample has to +be on the left hand side (LHS), followed by the scalar on the right hand side +(RHS). All bucket populations and the count and the sum of observations are +then divided by the scalar. A division by zero results in a histogram with no +regular buckets and the zero bucket population and the count and sum of +observations all set to +Inf, -Inf, or NaN, depending on their values in the +input histogram (positive, negative, or zero/NaN, respectively). For `/` with a +scalar on the LHS and a histogram sample on the RHS, and similarly for all +other arithmetic binary operators in any combination of a scalar and a +histogram sample, there is no result and the corresponding element is removed +from the resulting vector. Such a removal is flagged by an info-level +annotation. **Between two instant vectors**, a binary arithmetic operator is applied to -each entry in the left-hand side vector and its [matching element](#vector-matching) -in the right-hand vector. The result is propagated into the result vector with the -grouping labels becoming the output label set. The metric name is dropped. Entries -for which no matching entry in the right-hand vector can be found are not part of -the result. +each entry in the LHS vector and its [matching element](#vector-matching) in +the RHS vector. The result is propagated into the result vector with the +grouping labels becoming the output label set. Entries for which no matching +entry in the right-hand vector can be found are not part of the result. If two +float samples are matched, the behavior is obvious. If a float sample is +matched with a histogram sample, the behavior follows the same logic as between +a scalar and a histogram sample (see above), i.e. `*` and `/` (the latter with +the histogram sample on the LHS) are valid operations, while all others lead to +the removal of the corresponding element from the resulting vector. If two +histogram samples are matched, only `+` and `-` are valid operations, each +adding or substracting all matching bucket populations and the count and the +sum of observations. All other operations result in the removal of the +corresponding element from the output vector, flagged by an info-level +annotation. + +**In any arithmetic binary operation involving vectors**, the metric name is +dropped. ### Trigonometric binary operators @@ -46,9 +75,12 @@ The following trigonometric binary operators, which work in radians, exist in Pr * `atan2` (based on https://pkg.go.dev/math#Atan2) -Trigonometric operators allow trigonometric functions to be executed on two vectors using -vector matching, which isn't available with normal functions. They act in the same manner -as arithmetic operators. +Trigonometric operators allow trigonometric functions to be executed on two +vectors using vector matching, which isn't available with normal functions. +They act in the same manner as arithmetic operators. They only operate on float +samples. Operations involving histogram samples result in the removal of the +corresponding vector elements from the output vector, flagged by an +info-level annotation. ### Comparison binary operators @@ -72,20 +104,28 @@ operators result in another scalar that is either `0` (`false`) or `1` **Between an instant vector and a scalar**, these operators are applied to the value of every data sample in the vector, and vector elements between which the -comparison result is `false` get dropped from the result vector. If the `bool` -modifier is provided, vector elements that would be dropped instead have the value -`0` and vector elements that would be kept have the value `1`. The metric name -is dropped if the `bool` modifier is provided. +comparison result is `false` get dropped from the result vector. These +operation only work with float samples in the vector. For histogram samples, +the corresponding element is removed from the result vector, flagged by an +info-level annotation. **Between two instant vectors**, these operators behave as a filter by default, applied to matching entries. Vector elements for which the expression is not true or which do not find a match on the other side of the expression get dropped from the result, while the others are propagated into a result vector -with the grouping labels becoming the output label set. -If the `bool` modifier is provided, vector elements that would have been -dropped instead have the value `0` and vector elements that would be kept have -the value `1`, with the grouping labels again becoming the output label set. -The metric name is dropped if the `bool` modifier is provided. +with the grouping labels becoming the output label set. Matches between two +float samples work as usual, while matches between a float sample and a +histogram sample are invalid. The corresponding element is removed from the +result vector, flagged by an info-level annotation. Between two histogram +samples, `==` and `!=` work as expected, but all other comparison binary +operations are again invalid. + +**In any comparison binary operation involving vectors**, providing the `bool` +modifier changes the behavior in the following way: Vector elements that would +be dropped instead have the value `0` and vector elements that would be kept +have the value `1`. Additionally, the metric name is dropped. (Note that +invalid operations involving histogram samples still return no result rather +than the value `0`.) ### Logical/set binary operators @@ -108,6 +148,9 @@ which do not have matching label sets in `vector1`. `vector1` for which there are no elements in `vector2` with exactly matching label sets. All matching elements in both vectors are dropped. +As these logical/set binary operators do not interact with the sample values, +they work in the same way for float samples and histogram samples. + ## Vector matching Operations between vectors attempt to find a matching element in the right-hand side @@ -219,19 +262,20 @@ used to aggregate the elements of a single instant vector, resulting in a new vector of fewer elements with aggregated values: * `sum` (calculate sum over dimensions) +* `avg` (calculate the arithmetic average over dimensions) * `min` (select minimum over dimensions) * `max` (select maximum over dimensions) -* `avg` (calculate the average over dimensions) +* `bottomk` (smallest _k_ elements by sample value) +* `topk` (largest _k_ elements by sample value) +* `limitk` (sample _k_ elements, **experimental**, must be enabled with `--enable-feature=promql-experimental-functions`) +* `limit_ratio` (sample a pseudo-randem ratio _r_ of elements, **experimental**, must be enabled with `--enable-feature=promql-experimental-functions`) * `group` (all values in the resulting vector are 1) -* `stddev` (calculate population standard deviation over dimensions) -* `stdvar` (calculate population standard variance over dimensions) * `count` (count number of elements in the vector) * `count_values` (count number of elements with the same value) -* `bottomk` (smallest k elements by sample value) -* `topk` (largest k elements by sample value) + +* `stddev` (calculate population standard deviation over dimensions) +* `stdvar` (calculate population standard variance over dimensions) * `quantile` (calculate φ-quantile (0 ≤ φ ≤ 1) over dimensions) -* `limitk` (sample n elements) -* `limit_ratio` (sample elements with approximately 𝑟 ratio if `𝑟 > 0`, and the complement of such samples if `𝑟 = -(1.0 - 𝑟)`) These operators can either be used to aggregate over **all** label dimensions or preserve distinct dimensions by including a `without` or `by` clause. These @@ -251,29 +295,67 @@ all other labels are preserved in the output. `by` does the opposite and drops labels that are not listed in the `by` clause, even if their label values are identical between all elements of the vector. -`parameter` is only required for `count_values`, `quantile`, `topk`, -`bottomk`, `limitk` and `limit_ratio`. +`parameter` is only required for `topk`, `bottomk`, `limitk`, `limit_ratio`, +`quantile`, and `count_values`. It is used as the value for _k_, _r_, φ, or the +name of the additional label, respectively. + +### Detailed explanations + +`sum` sums up sample values in the same way as the `+` binary operator does +between two values. Similarly, `avg` divides the sum by the number of +aggregated samples in the same way as the `/` binary operator. Therefore, all +sample values aggregation into a single resulting vector element must either be +float samples or histogram samples. An aggregation of a mix of both is invalid, +resulting in the removeal of the corresponding vector element from the output +vector, flagged by a warn-level annotation. + +`min` and `max` only operate on float samples, following IEEE 754 floating +point arithmetic, which in particular implies that `NaN` is only ever +considered a minimum or maximum if all aggregated values are `NaN`. Histogram +samples in the input vector are ignored, flagged by an info-level annotation. + +`topk` and `bottomk` are different from other aggregators in that a subset of +the input samples, including the original labels, are returned in the result +vector. `by` and `without` are only used to bucket the input vector. Similar to +`min` and `max`, they only operate on float samples, considering `NaN` values +to be farthest from the top or bottom, respectively. Histogram samples in the +input vector are ignored, flagged by an info-level annotation. + +`limitk` and `limit_ratio` also return a subset of the input samples, including +the original labels in the result vector. The subset is selected in a +deterministic pseudo-random way. `limitk` picks _k_ samples, while +`limit_ratio` picks a ratio _r_ of samples (each determined by `parameter`). +This happens independent of the sample type. Therefore, it works for both float +samples and histogram samples. _r_ can be between +1 and -1. The absolute value +of _r_ is used as the selection ratio, but the selection order is inverted for +a negative _r_, which can be used to select complements. For example, +`limit_ratio(0.1, ...)` returns a deterministic set of approximatiely 10% of +the input samples, while `limit_ratio(-0.9, ...)` returns precisely the +remaining approximately 90% of the input samples not returned by +`limit_ratio(0.1, ...)`. + +`group` and `count` do not do not interact with the sample values, +they work in the same way for float samples and histogram samples. `count_values` outputs one time series per unique sample value. Each series has an additional label. The name of that label is given by the aggregation parameter, and the label value is the unique sample value. The value of each time series is the number of times that sample value was present. +`count_values` works with both float samples and histogram samples. For the +latter, a compact string representation of the histogram sample value is used +as the label value. -`topk` and `bottomk` are different from other aggregators in that a subset of -the input samples, including the original labels, are returned in the result -vector. `by` and `without` are only used to bucket the input vector. - -`limitk` and `limit_ratio` also return a subset of the input samples, -including the original labels in the result vector, these are experimental -operators that must be enabled with `--enable-feature=promql-experimental-functions`. +`stddev` and `stdvar` only work with float samples, following IEEE 754 floating +point arithmetic. Histogram samples in the input vector are ignored, flagged by +an info-level annotation. `quantile` calculates the φ-quantile, the value that ranks at number φ*N among the N metric values of the dimensions aggregated over. φ is provided as the aggregation parameter. For example, `quantile(0.5, ...)` calculates the median, -`quantile(0.95, ...)` the 95th percentile. For φ = `NaN`, `NaN` is returned. For φ < 0, `-Inf` is returned. For φ > 1, `+Inf` is returned. +`quantile(0.95, ...)` the 95th percentile. For φ = `NaN`, `NaN` is returned. +For φ < 0, `-Inf` is returned. For φ > 1, `+Inf` is returned. - -Example: +### Examples If the metric `http_requests_total` had time series that fan out by `application`, `instance`, and `group` labels, we could calculate the total @@ -303,28 +385,6 @@ could write: limitk(10, http_requests_total) -To deterministically sample approximately 10% of timeseries we could write: - - limit_ratio(0.1, http_requests_total) - -Given that `limit_ratio()` implements a deterministic sampling algorithm (based -on labels' hash), you can get the _complement_ of the above samples, i.e. -approximately 90%, but precisely those not returned by `limit_ratio(0.1, ...)` -with: - - limit_ratio(-0.9, http_requests_total) - -You can also use this feature to e.g. verify that `avg()` is a representative -aggregation for your samples' values, by checking that the difference between -averaging two samples' subsets is "small" when compared to the standard -deviation. - - abs( - avg(limit_ratio(0.5, http_requests_total)) - - - avg(limit_ratio(-0.5, http_requests_total)) - ) <= bool stddev(http_requests_total) - ## Binary operator precedence The following list shows the precedence of binary operators in Prometheus, from @@ -340,35 +400,3 @@ highest to lowest. Operators on the same precedence level are left-associative. For example, `2 * 3 % 2` is equivalent to `(2 * 3) % 2`. However `^` is right associative, so `2 ^ 3 ^ 2` is equivalent to `2 ^ (3 ^ 2)`. - -## Operators for native histograms - -Native histograms are an experimental feature. Ingesting native histograms has -to be enabled via a [feature flag](../../feature_flags.md#native-histograms). Once -native histograms have been ingested, they can be queried (even after the -feature flag has been disabled again). However, the operator support for native -histograms is still very limited. - -Logical/set binary operators work as expected even if histogram samples are -involved. They only check for the existence of a vector element and don't -change their behavior depending on the sample type of an element (float or -histogram). The `count` aggregation operator works similarly. - -The binary `+` and `-` operators between two native histograms and the `sum` -and `avg` aggregation operators to aggregate native histograms are fully -supported. Even if the histograms involved have different bucket layouts, the -buckets are automatically converted appropriately so that the operation can be -performed. (With the currently supported bucket schemas, that's always -possible.) If either operator has to aggregate a mix of histogram samples and -float samples, the corresponding vector element is removed from the output -vector entirely. - -The binary `*` operator works between a native histogram and a float in any -order, while the binary `/` operator can be used between a native histogram -and a float in that exact order. - -All other operators (and unmentioned cases for the above operators) do not -behave in a meaningful way. They either treat the histogram sample as if it -were a float sample of value 0, or (in case of arithmetic operations between a -scalar and a vector) they leave the histogram sample unchanged. This behavior -will change to a meaningful one before native histograms are a stable feature. diff --git a/documentation/examples/remote_storage/go.mod b/documentation/examples/remote_storage/go.mod index da8ef55332..541a8b382d 100644 --- a/documentation/examples/remote_storage/go.mod +++ b/documentation/examples/remote_storage/go.mod @@ -8,7 +8,7 @@ require ( github.com/golang/snappy v0.0.4 github.com/influxdata/influxdb-client-go/v2 v2.14.0 github.com/prometheus/client_golang v1.20.5 - github.com/prometheus/common v0.61.0 + github.com/prometheus/common v0.62.0 github.com/prometheus/prometheus v1.99.0 github.com/stretchr/testify v1.10.0 ) @@ -66,7 +66,7 @@ require ( golang.org/x/time v0.5.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 // indirect google.golang.org/grpc v1.65.0 // indirect - google.golang.org/protobuf v1.35.2 // indirect + google.golang.org/protobuf v1.36.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apimachinery v0.29.3 // indirect diff --git a/documentation/examples/remote_storage/go.sum b/documentation/examples/remote_storage/go.sum index 53077c877a..bea6b8e68b 100644 --- a/documentation/examples/remote_storage/go.sum +++ b/documentation/examples/remote_storage/go.sum @@ -273,8 +273,8 @@ github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.61.0 h1:3gv/GThfX0cV2lpO7gkTUwZru38mxevy90Bj8YFSRQQ= -github.com/prometheus/common v0.61.0/go.mod h1:zr29OCN/2BsJRaFwG8QOBr41D6kkchKbpeNH7pAjb/s= +github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= +github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= github.com/prometheus/common/sigv4 v0.1.0 h1:qoVebwtwwEhS85Czm2dSROY5fTo2PAPEVdDeppTwGX4= github.com/prometheus/common/sigv4 v0.1.0/go.mod h1:2Jkxxk9yYvCkE5G1sQT7GuEXm57JrvHu9k5YwTjsNtI= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -421,8 +421,8 @@ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miE google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io= -google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= +google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/documentation/prometheus-mixin/dashboards.libsonnet b/documentation/prometheus-mixin/dashboards.libsonnet index 22b8c92e6e..50cbf72d1d 100644 --- a/documentation/prometheus-mixin/dashboards.libsonnet +++ b/documentation/prometheus-mixin/dashboards.libsonnet @@ -56,7 +56,7 @@ local row = panel.row; + variable.query.selectionOptions.withIncludeAll(true, '.+') + variable.query.selectionOptions.withMulti(true) + if showMultiCluster then - variable.query.queryTypes.withLabelValues('job', metric='prometheus_build_info{cluster=~"$cluster"}') + variable.query.queryTypes.withLabelValues('job', metric='prometheus_build_info{%(clusterLabel)s=~"$cluster"}' % $._config) else variable.query.queryTypes.withLabelValues('job', metric='prometheus_build_info{%(prometheusSelector)s}' % $._config) ; @@ -70,7 +70,7 @@ local row = panel.row; + variable.query.selectionOptions.withIncludeAll(true, '.+') + variable.query.selectionOptions.withMulti(true) + if showMultiCluster then - variable.query.queryTypes.withLabelValues('instance', metric='prometheus_build_info{cluster=~"$cluster", job=~"$job"}') + variable.query.queryTypes.withLabelValues('instance', metric='prometheus_build_info{%(clusterLabel)s=~"$cluster", job=~"$job"}' % $._config) else variable.query.queryTypes.withLabelValues('instance', metric='prometheus_build_info{job=~"$job"}') ; @@ -121,14 +121,14 @@ local row = panel.row; panel.table.queryOptions.withTargets([ prometheus.new( '$datasource', - 'count by (cluster, job, instance, version) (prometheus_build_info{cluster=~"$cluster", job=~"$job", instance=~"$instance"})' + 'count by (cluster, job, instance, version) (prometheus_build_info{%(clusterLabel)s=~"$cluster", job=~"$job", instance=~"$instance"})' % $._config ) + prometheus.withFormat('table') + prometheus.withInstant(true) + prometheus.withLegendFormat(''), prometheus.new( '$datasource', - 'max by (cluster, job, instance) (time() - process_start_time_seconds{cluster=~"$cluster", job=~"$job", instance=~"$instance"})' + 'max by (cluster, job, instance) (time() - process_start_time_seconds{%(clusterLabel)s=~"$cluster", job=~"$job", instance=~"$instance"})' % $._config ) + prometheus.withFormat('table') + prometheus.withInstant(true) @@ -163,10 +163,10 @@ local row = panel.row; panel.timeSeries.queryOptions.withTargets([ prometheus.new( '$datasource', - 'sum(rate(prometheus_target_sync_length_seconds_sum{cluster=~"$cluster",job=~"$job",instance=~"$instance"}[5m])) by (cluster, job, scrape_job, instance) * 1e3' + 'sum(rate(prometheus_target_sync_length_seconds_sum{%(clusterLabel)s=~"$cluster",job=~"$job",instance=~"$instance"}[5m])) by (%(clusterLabel)s, job, scrape_job, instance) * 1e3' % $._config ) + prometheus.withFormat('time_series') - + prometheus.withLegendFormat('{{cluster}}:{{job}}:{{instance}}:{{scrape_job}}'), + + prometheus.withLegendFormat('{{%(clusterLabel)s}}:{{job}}:{{instance}}:{{scrape_job}}' % $._config), ]) else panel.timeSeries.queryOptions.withTargets([ @@ -190,10 +190,10 @@ local row = panel.row; panel.timeSeries.queryOptions.withTargets([ prometheus.new( '$datasource', - 'sum by (cluster, job, instance) (prometheus_sd_discovered_targets{cluster=~"$cluster", job=~"$job",instance=~"$instance"})' + 'sum by (%(clusterLabel)s, job, instance) (prometheus_sd_discovered_targets{%(clusterLabel)s=~"$cluster", job=~"$job",instance=~"$instance"})' % $._config ) + prometheus.withFormat('time_series') - + prometheus.withLegendFormat('{{cluster}}:{{job}}:{{instance}}'), + + prometheus.withLegendFormat('{{%(clusterLabel)s}}:{{job}}:{{instance}}' % $._config), ]) else panel.timeSeries.queryOptions.withTargets([ @@ -216,10 +216,10 @@ local row = panel.row; panel.timeSeries.queryOptions.withTargets([ prometheus.new( '$datasource', - 'rate(prometheus_target_interval_length_seconds_sum{cluster=~"$cluster", job=~"$job",instance=~"$instance"}[5m]) / rate(prometheus_target_interval_length_seconds_count{cluster=~"$cluster", job=~"$job",instance=~"$instance"}[5m]) * 1e3' + 'rate(prometheus_target_interval_length_seconds_sum{%(clusterLabel)s=~"$cluster", job=~"$job",instance=~"$instance"}[5m]) / rate(prometheus_target_interval_length_seconds_count{%(clusterLabel)s=~"$cluster", job=~"$job",instance=~"$instance"}[5m]) * 1e3' % $._config ) + prometheus.withFormat('time_series') - + prometheus.withLegendFormat('{{cluster}}:{{job}}:{{instance}} {{interval}} configured'), + + prometheus.withLegendFormat('{{%(clusterLabel)s}}:{{job}}:{{instance}} {{interval}} configured' % $._config), ]) else panel.timeSeries.queryOptions.withTargets([ @@ -243,34 +243,34 @@ local row = panel.row; panel.timeSeries.queryOptions.withTargets([ prometheus.new( '$datasource', - 'sum by (cluster, job, instance) (rate(prometheus_target_scrapes_exceeded_body_size_limit_total{cluster=~"$cluster",job=~"$job",instance=~"$instance"}[1m]))' + 'sum by (%(clusterLabel)s, job, instance) (rate(prometheus_target_scrapes_exceeded_body_size_limit_total{%(clusterLabel)s=~"$cluster",job=~"$job",instance=~"$instance"}[1m]))' % $._config ) + prometheus.withFormat('time_series') - + prometheus.withLegendFormat('exceeded body size limit: {{cluster}} {{job}} {{instance}}'), + + prometheus.withLegendFormat('exceeded body size limit: {{%(clusterLabel)s}} {{job}} {{instance}}' % $._config), prometheus.new( '$datasource', - 'sum by (cluster, job, instance) (rate(prometheus_target_scrapes_exceeded_sample_limit_total{cluster=~"$cluster",job=~"$job",instance=~"$instance"}[1m]))' + 'sum by (%(clusterLabel)s, job, instance) (rate(prometheus_target_scrapes_exceeded_sample_limit_total{%(clusterLabel)s=~"$cluster",job=~"$job",instance=~"$instance"}[1m]))' % $._config ) + prometheus.withFormat('time_series') - + prometheus.withLegendFormat('exceeded sample limit: {{cluster}} {{job}} {{instance}}'), + + prometheus.withLegendFormat('exceeded sample limit: {{%(clusterLabel)s}} {{job}} {{instance}}' % $._config), prometheus.new( '$datasource', - 'sum by (cluster, job, instance) (rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{cluster=~"$cluster",job=~"$job",instance=~"$instance"}[1m]))' + 'sum by (%(clusterLabel)s, job, instance) (rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{%(clusterLabel)s=~"$cluster",job=~"$job",instance=~"$instance"}[1m]))' % $._config ) + prometheus.withFormat('time_series') - + prometheus.withLegendFormat('duplicate timestamp: {{cluster}} {{job}} {{instance}}'), + + prometheus.withLegendFormat('duplicate timestamp: {{%(clusterLabel)s}} {{job}} {{instance}}' % $._config), prometheus.new( '$datasource', - 'sum by (cluster, job, instance) (rate(prometheus_target_scrapes_sample_out_of_bounds_total{cluster=~"$cluster",job=~"$job",instance=~"$instance"}[1m]))' + 'sum by (%(clusterLabel)s, job, instance) (rate(prometheus_target_scrapes_sample_out_of_bounds_total{%(clusterLabel)s=~"$cluster",job=~"$job",instance=~"$instance"}[1m]))' % $._config ) + prometheus.withFormat('time_series') - + prometheus.withLegendFormat('out of bounds: {{cluster}} {{job}} {{instance}}'), + + prometheus.withLegendFormat('out of bounds: {{%(clusterLabel)s}} {{job}} {{instance}}' % $._config), prometheus.new( '$datasource', - 'sum by (cluster, job, instance) (rate(prometheus_target_scrapes_sample_out_of_order_total{cluster=~"$cluster",job=~"$job",instance=~"$instance"}[1m]))' + 'sum by (%(clusterLabel)s, job, instance) (rate(prometheus_target_scrapes_sample_out_of_order_total{%(clusterLabel)s=~"$cluster",job=~"$job",instance=~"$instance"}[1m]))' % $._config ) + prometheus.withFormat('time_series') - + prometheus.withLegendFormat('out of order: {{cluster}} {{job}} {{instance}}'), + + prometheus.withLegendFormat('out of order: {{%(clusterLabel)s}} {{job}} {{instance}}' % $._config), ]) else panel.timeSeries.queryOptions.withTargets([ @@ -318,10 +318,10 @@ local row = panel.row; panel.timeSeries.queryOptions.withTargets([ prometheus.new( '$datasource', - 'rate(prometheus_tsdb_head_samples_appended_total{cluster=~"$cluster", job=~"$job",instance=~"$instance"}[5m])' + 'rate(prometheus_tsdb_head_samples_appended_total{%(clusterLabel)s=~"$cluster", job=~"$job",instance=~"$instance"}[5m])' % $._config ) + prometheus.withFormat('time_series') - + prometheus.withLegendFormat('{{cluster}} {{job}} {{instance}}'), + + prometheus.withLegendFormat('{{%(clusterLabel)s}} {{job}} {{instance}}' % $._config), ]) else panel.timeSeries.queryOptions.withTargets([ @@ -345,10 +345,10 @@ local row = panel.row; panel.timeSeries.queryOptions.withTargets([ prometheus.new( '$datasource', - 'prometheus_tsdb_head_series{cluster=~"$cluster",job=~"$job",instance=~"$instance"}' + 'prometheus_tsdb_head_series{%(clusterLabel)s=~"$cluster",job=~"$job",instance=~"$instance"}' % $._config ) + prometheus.withFormat('time_series') - + prometheus.withLegendFormat('{{cluster}} {{job}} {{instance}} head series'), + + prometheus.withLegendFormat('{{%(clusterLabel)s}} {{job}} {{instance}} head series' % $._config), ]) else panel.timeSeries.queryOptions.withTargets([ @@ -372,10 +372,10 @@ local row = panel.row; panel.timeSeries.queryOptions.withTargets([ prometheus.new( '$datasource', - 'prometheus_tsdb_head_chunks{cluster=~"$cluster",job=~"$job",instance=~"$instance"}' + 'prometheus_tsdb_head_chunks{%(clusterLabel)s=~"$cluster",job=~"$job",instance=~"$instance"}' % $._config ) + prometheus.withFormat('time_series') - + prometheus.withLegendFormat('{{cluster}} {{job}} {{instance}} head chunks'), + + prometheus.withLegendFormat('{{%(clusterLabel)s}} {{job}} {{instance}} head chunks' % $._config), ]) else panel.timeSeries.queryOptions.withTargets([ @@ -399,10 +399,10 @@ local row = panel.row; panel.timeSeries.queryOptions.withTargets([ prometheus.new( '$datasource', - 'rate(prometheus_engine_query_duration_seconds_count{cluster=~"$cluster",job=~"$job",instance=~"$instance",slice="inner_eval"}[5m])' + 'rate(prometheus_engine_query_duration_seconds_count{%(clusterLabel)s=~"$cluster",job=~"$job",instance=~"$instance",slice="inner_eval"}[5m])' % $._config ) + prometheus.withFormat('time_series') - + prometheus.withLegendFormat('{{cluster}} {{job}} {{instance}}'), + + prometheus.withLegendFormat('{{%(clusterLabel)s}} {{job}} {{instance}}' % $._config), ]) else panel.timeSeries.queryOptions.withTargets([ @@ -426,7 +426,7 @@ local row = panel.row; panel.timeSeries.queryOptions.withTargets([ prometheus.new( '$datasource', - 'max by (slice) (prometheus_engine_query_duration_seconds{quantile="0.9",cluster=~"$cluster", job=~"$job",instance=~"$instance"}) * 1e3' + 'max by (slice) (prometheus_engine_query_duration_seconds{quantile="0.9",%(clusterLabel)s=~"$cluster", job=~"$job",instance=~"$instance"}) * 1e3' % $._config ) + prometheus.withFormat('time_series') + prometheus.withLegendFormat('{{slice}}'), @@ -514,7 +514,7 @@ local row = panel.row; + variable.query.withDatasourceFromVariable(datasourceVariable) + variable.query.refresh.onTime() + variable.query.selectionOptions.withIncludeAll(true) - + variable.query.queryTypes.withLabelValues('instance', metric='prometheus_build_info{cluster=~"$cluster"}') + + variable.query.queryTypes.withLabelValues('instance', metric='prometheus_build_info{%(clusterLabel)s=~"$cluster"}' % $._config) ; local urlVariable = @@ -522,7 +522,7 @@ local row = panel.row; + variable.query.withDatasourceFromVariable(datasourceVariable) + variable.query.refresh.onTime() + variable.query.selectionOptions.withIncludeAll(true) - + variable.query.queryTypes.withLabelValues('url', metric='prometheus_remote_storage_shards{cluster=~"$cluster", instance=~"$instance"}') + + variable.query.queryTypes.withLabelValues('url', metric='prometheus_remote_storage_shards{%(clusterLabel)s=~"$cluster", instance=~"$instance"}' % $._config) ; local timestampComparison = @@ -534,15 +534,15 @@ local row = panel.row; '$datasource', ||| ( - prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~"$cluster", instance=~"$instance"} + prometheus_remote_storage_highest_timestamp_in_seconds{%(clusterLabel)s=~"$cluster", instance=~"$instance"} - - ignoring(remote_name, url) group_right(instance) (prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~"$cluster", instance=~"$instance", url=~"$url"} != 0) + ignoring(remote_name, url) group_right(instance) (prometheus_remote_storage_queue_highest_sent_timestamp_seconds{%(clusterLabel)s=~"$cluster", instance=~"$instance", url=~"$url"} != 0) ) - ||| + ||| % $._config ) + prometheus.withFormat('time_series') + prometheus.withIntervalFactor(2) - + prometheus.withLegendFormat('{{cluster}}:{{instance}} {{remote_name}}:{{url}}'), + + prometheus.withLegendFormat('{{%(clusterLabel)s}}::{{instance}} {{remote_name}}:{{url}}' % $._config), ]); local timestampComparisonRate = @@ -554,15 +554,15 @@ local row = panel.row; '$datasource', ||| clamp_min( - rate(prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~"$cluster", instance=~"$instance"}[5m]) + rate(prometheus_remote_storage_highest_timestamp_in_seconds{%(clusterLabel)s=~"$cluster", instance=~"$instance"}[5m]) - - ignoring (remote_name, url) group_right(instance) rate(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~"$cluster", instance=~"$instance", url=~"$url"}[5m]) + ignoring (remote_name, url) group_right(instance) rate(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{%(clusterLabel)s=~"$cluster", instance=~"$instance", url=~"$url"}[5m]) , 0) - ||| + ||| % $._config ) + prometheus.withFormat('time_series') + prometheus.withIntervalFactor(2) - + prometheus.withLegendFormat('{{cluster}}:{{instance}} {{remote_name}}:{{url}}'), + + prometheus.withLegendFormat('{{%(clusterLabel)s}}:{{instance}} {{remote_name}}:{{url}}' % $._config), ]); local samplesRate = @@ -574,16 +574,16 @@ local row = panel.row; '$datasource', ||| rate( - prometheus_remote_storage_samples_in_total{cluster=~"$cluster", instance=~"$instance"}[5m]) + prometheus_remote_storage_samples_in_total{%(clusterLabel)s=~"$cluster", instance=~"$instance"}[5m]) - - ignoring(remote_name, url) group_right(instance) (rate(prometheus_remote_storage_succeeded_samples_total{cluster=~"$cluster", instance=~"$instance", url=~"$url"}[5m]) or rate(prometheus_remote_storage_samples_total{cluster=~"$cluster", instance=~"$instance", url=~"$url"}[5m])) + ignoring(remote_name, url) group_right(instance) (rate(prometheus_remote_storage_succeeded_samples_total{%(clusterLabel)s=~"$cluster", instance=~"$instance", url=~"$url"}[5m]) or rate(prometheus_remote_storage_samples_total{%(clusterLabel)s=~"$cluster", instance=~"$instance", url=~"$url"}[5m])) - - (rate(prometheus_remote_storage_dropped_samples_total{cluster=~"$cluster", instance=~"$instance", url=~"$url"}[5m]) or rate(prometheus_remote_storage_samples_dropped_total{cluster=~"$cluster", instance=~"$instance", url=~"$url"}[5m])) - ||| + (rate(prometheus_remote_storage_dropped_samples_total{%(clusterLabel)s=~"$cluster", instance=~"$instance", url=~"$url"}[5m]) or rate(prometheus_remote_storage_samples_dropped_total{%(clusterLabel)s=~"$cluster", instance=~"$instance", url=~"$url"}[5m])) + ||| % $._config ) + prometheus.withFormat('time_series') + prometheus.withIntervalFactor(2) - + prometheus.withLegendFormat('{{cluster}}:{{instance}} {{remote_name}}:{{url}}'), + + prometheus.withLegendFormat('{{%(clusterLabel)s}}:{{instance}} {{remote_name}}:{{url}}' % $._config), ]); local currentShards = @@ -593,11 +593,11 @@ local row = panel.row; + panel.timeSeries.queryOptions.withTargets([ prometheus.new( '$datasource', - 'prometheus_remote_storage_shards{cluster=~"$cluster", instance=~"$instance", url=~"$url"}' + 'prometheus_remote_storage_shards{%(clusterLabel)s=~"$cluster", instance=~"$instance", url=~"$url"}' % $._config ) + prometheus.withFormat('time_series') + prometheus.withIntervalFactor(2) - + prometheus.withLegendFormat('{{cluster}}:{{instance}} {{remote_name}}:{{url}}'), + + prometheus.withLegendFormat('{{%(clusterLabel)s}}:{{instance}} {{remote_name}}:{{url}}' % $._config), ]); local maxShards = @@ -607,11 +607,11 @@ local row = panel.row; + panel.timeSeries.queryOptions.withTargets([ prometheus.new( '$datasource', - 'prometheus_remote_storage_shards_max{cluster=~"$cluster", instance=~"$instance", url=~"$url"}' + 'prometheus_remote_storage_shards_max{%(clusterLabel)s=~"$cluster", instance=~"$instance", url=~"$url"}' % $._config ) + prometheus.withFormat('time_series') + prometheus.withIntervalFactor(2) - + prometheus.withLegendFormat('{{cluster}}:{{instance}} {{remote_name}}:{{url}}'), + + prometheus.withLegendFormat('{{%(clusterLabel)s}}:{{instance}} {{remote_name}}:{{url}}' % $._config), ]); local minShards = @@ -621,11 +621,11 @@ local row = panel.row; + panel.timeSeries.queryOptions.withTargets([ prometheus.new( '$datasource', - 'prometheus_remote_storage_shards_min{cluster=~"$cluster", instance=~"$instance", url=~"$url"}' + 'prometheus_remote_storage_shards_min{%(clusterLabel)s=~"$cluster", instance=~"$instance", url=~"$url"}' % $._config ) + prometheus.withFormat('time_series') + prometheus.withIntervalFactor(2) - + prometheus.withLegendFormat('{{cluster}}:{{instance}} {{remote_name}}:{{url}}'), + + prometheus.withLegendFormat('{{%(clusterLabel)s}}{{instance}} {{remote_name}}:{{url}}' % $._config), ]); local desiredShards = @@ -635,11 +635,11 @@ local row = panel.row; + panel.timeSeries.queryOptions.withTargets([ prometheus.new( '$datasource', - 'prometheus_remote_storage_shards_desired{cluster=~"$cluster", instance=~"$instance", url=~"$url"}' + 'prometheus_remote_storage_shards_desired{%(clusterLabel)s=~"$cluster", instance=~"$instance", url=~"$url"}' % $._config ) + prometheus.withFormat('time_series') + prometheus.withIntervalFactor(2) - + prometheus.withLegendFormat('{{cluster}}:{{instance}} {{remote_name}}:{{url}}'), + + prometheus.withLegendFormat('{{%(clusterLabel)s}}:{{instance}} {{remote_name}}:{{url}}' % $._config), ]); local shardsCapacity = @@ -649,11 +649,11 @@ local row = panel.row; + panel.timeSeries.queryOptions.withTargets([ prometheus.new( '$datasource', - 'prometheus_remote_storage_shard_capacity{cluster=~"$cluster", instance=~"$instance", url=~"$url"}' + 'prometheus_remote_storage_shard_capacity{%(clusterLabel)s=~"$cluster", instance=~"$instance", url=~"$url"}' % $._config ) + prometheus.withFormat('time_series') + prometheus.withIntervalFactor(2) - + prometheus.withLegendFormat('{{cluster}}:{{instance}} {{remote_name}}:{{url}}'), + + prometheus.withLegendFormat('{{%(clusterLabel)s}}:{{instance}} {{remote_name}}:{{url}}' % $._config), ]); local pendingSamples = @@ -663,11 +663,11 @@ local row = panel.row; + panel.timeSeries.queryOptions.withTargets([ prometheus.new( '$datasource', - 'prometheus_remote_storage_pending_samples{cluster=~"$cluster", instance=~"$instance", url=~"$url"} or prometheus_remote_storage_samples_pending{cluster=~"$cluster", instance=~"$instance", url=~"$url"}' + 'prometheus_remote_storage_pending_samples{%(clusterLabel)s=~"$cluster", instance=~"$instance", url=~"$url"} or prometheus_remote_storage_samples_pending{%(clusterLabel)s=~"$cluster", instance=~"$instance", url=~"$url"}' % $._config ) + prometheus.withFormat('time_series') + prometheus.withIntervalFactor(2) - + prometheus.withLegendFormat('{{cluster}}:{{instance}} {{remote_name}}:{{url}}'), + + prometheus.withLegendFormat('{{%(clusterLabel)s}}:{{instance}} {{remote_name}}:{{url}}' % $._config), ]); local walSegment = @@ -679,11 +679,11 @@ local row = panel.row; + panel.timeSeries.queryOptions.withTargets([ prometheus.new( '$datasource', - 'prometheus_tsdb_wal_segment_current{cluster=~"$cluster", instance=~"$instance"}' + 'prometheus_tsdb_wal_segment_current{%(clusterLabel)s=~"$cluster", instance=~"$instance"}' % $._config ) + prometheus.withFormat('time_series') + prometheus.withIntervalFactor(2) - + prometheus.withLegendFormat('{{cluster}}:{{instance}}'), + + prometheus.withLegendFormat('{{%(clusterLabel)s}}:{{instance}}' % $._config), ]); local queueSegment = @@ -695,11 +695,11 @@ local row = panel.row; + panel.timeSeries.queryOptions.withTargets([ prometheus.new( '$datasource', - 'prometheus_wal_watcher_current_segment{cluster=~"$cluster", instance=~"$instance"}' + 'prometheus_wal_watcher_current_segment{%(clusterLabel)s=~"$cluster", instance=~"$instance"}' % $._config ) + prometheus.withFormat('time_series') + prometheus.withIntervalFactor(2) - + prometheus.withLegendFormat('{{cluster}}:{{instance}} {{consumer}}'), + + prometheus.withLegendFormat('{{%(clusterLabel)s}}:{{instance}} {{consumer}}' % $._config), ]); local droppedSamples = @@ -710,11 +710,11 @@ local row = panel.row; + panel.timeSeries.queryOptions.withTargets([ prometheus.new( '$datasource', - 'rate(prometheus_remote_storage_dropped_samples_total{cluster=~"$cluster", instance=~"$instance", url=~"$url"}[5m]) or rate(prometheus_remote_storage_samples_dropped_total{cluster=~"$cluster", instance=~"$instance", url=~"$url"}[5m])' + 'rate(prometheus_remote_storage_dropped_samples_total{%(clusterLabel)s=~"$cluster", instance=~"$instance", url=~"$url"}[5m]) or rate(prometheus_remote_storage_samples_dropped_total{%(clusterLabel)s=~"$cluster", instance=~"$instance", url=~"$url"}[5m])' % $._config ) + prometheus.withFormat('time_series') + prometheus.withIntervalFactor(2) - + prometheus.withLegendFormat('{{cluster}}:{{instance}} {{remote_name}}:{{url}}'), + + prometheus.withLegendFormat('{{%(clusterLabel)s}}:{{instance}} {{remote_name}}:{{url}}' % $._config), ]); local failedSamples = @@ -725,11 +725,11 @@ local row = panel.row; + panel.timeSeries.queryOptions.withTargets([ prometheus.new( '$datasource', - 'rate(prometheus_remote_storage_failed_samples_total{cluster=~"$cluster", instance=~"$instance", url=~"$url"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{cluster=~"$cluster", instance=~"$instance", url=~"$url"}[5m])' + 'rate(prometheus_remote_storage_failed_samples_total{%(clusterLabel)s=~"$cluster", instance=~"$instance", url=~"$url"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{%(clusterLabel)s=~"$cluster", instance=~"$instance", url=~"$url"}[5m])' % $._config ) + prometheus.withFormat('time_series') + prometheus.withIntervalFactor(2) - + prometheus.withLegendFormat('{{cluster}}:{{instance}} {{remote_name}}:{{url}}'), + + prometheus.withLegendFormat('{{%(clusterLabel)s}}:{{instance}} {{remote_name}}:{{url}}' % $._config), ]); local retriedSamples = @@ -740,11 +740,11 @@ local row = panel.row; + panel.timeSeries.queryOptions.withTargets([ prometheus.new( '$datasource', - 'rate(prometheus_remote_storage_retried_samples_total{cluster=~"$cluster", instance=~"$instance", url=~"$url"}[5m]) or rate(prometheus_remote_storage_samples_retried_total{cluster=~"$cluster", instance=~"$instance", url=~"$url"}[5m])' + 'rate(prometheus_remote_storage_retried_samples_total{%(clusterLabel)s=~"$cluster", instance=~"$instance", url=~"$url"}[5m]) or rate(prometheus_remote_storage_samples_retried_total{%(clusterLabel)s=~"$cluster", instance=~"$instance", url=~"$url"}[5m])' % $._config ) + prometheus.withFormat('time_series') + prometheus.withIntervalFactor(2) - + prometheus.withLegendFormat('{{cluster}}:{{instance}} {{remote_name}}:{{url}}'), + + prometheus.withLegendFormat('{{%(clusterLabel)s}}:{{instance}} {{remote_name}}:{{url}}' % $._config), ]); local enqueueRetries = @@ -755,11 +755,11 @@ local row = panel.row; + panel.timeSeries.queryOptions.withTargets([ prometheus.new( '$datasource', - 'rate(prometheus_remote_storage_enqueue_retries_total{cluster=~"$cluster", instance=~"$instance", url=~"$url"}[5m])' + 'rate(prometheus_remote_storage_enqueue_retries_total{%(clusterLabel)s=~"$cluster", instance=~"$instance", url=~"$url"}[5m])' % $._config ) + prometheus.withFormat('time_series') + prometheus.withIntervalFactor(2) - + prometheus.withLegendFormat('{{cluster}}:{{instance}} {{remote_name}}:{{url}}'), + + prometheus.withLegendFormat('{{%(clusterLabel)s}}:{{instance}} {{remote_name}}:{{url}}' % $._config), ]); dashboard.new('%(prefix)sRemote Write' % $._config.grafanaPrometheus) diff --git a/go.mod b/go.mod index fbbeaf19e3..7509c5e2ee 100644 --- a/go.mod +++ b/go.mod @@ -17,8 +17,8 @@ require ( github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 github.com/cespare/xxhash/v2 v2.3.0 github.com/dennwc/varint v1.0.0 - github.com/digitalocean/godo v1.132.0 - github.com/docker/docker v27.4.1+incompatible + github.com/digitalocean/godo v1.136.0 + github.com/docker/docker v27.5.1+incompatible github.com/edsrzf/mmap-go v1.2.0 github.com/envoyproxy/go-control-plane/envoy v1.32.3 github.com/envoyproxy/protoc-gen-validate v1.2.1 @@ -36,19 +36,19 @@ require ( github.com/grpc-ecosystem/grpc-gateway v1.16.0 github.com/hashicorp/consul/api v1.31.0 github.com/hashicorp/nomad/api v0.0.0-20241218080744-e3ac00f30eec - github.com/hetznercloud/hcloud-go/v2 v2.18.0 + github.com/hetznercloud/hcloud-go/v2 v2.19.0 github.com/ionos-cloud/sdk-go/v6 v6.3.2 github.com/json-iterator/go v1.1.12 github.com/klauspost/compress v1.17.11 github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b - github.com/linode/linodego v1.46.0 + github.com/linode/linodego v1.47.0 github.com/miekg/dns v1.1.63 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f github.com/nsf/jsondiff v0.0.0-20230430225905-43f6cf3098c1 github.com/oklog/run v1.1.0 github.com/oklog/ulid v1.3.1 - github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor v0.116.0 + github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor v0.118.0 github.com/ovh/go-ovh v1.6.0 github.com/prometheus/alertmanager v0.28.0 github.com/prometheus/client_golang v1.21.0-rc.0 @@ -56,7 +56,7 @@ require ( github.com/prometheus/common v0.62.0 github.com/prometheus/common/assets v0.2.0 github.com/prometheus/exporter-toolkit v0.13.2 - github.com/prometheus/sigv4 v0.1.1 + github.com/prometheus/sigv4 v0.1.2 github.com/scaleway/scaleway-sdk-go v1.0.0-beta.30 github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c github.com/stretchr/testify v1.10.0 @@ -84,7 +84,7 @@ require ( golang.org/x/sys v0.29.0 golang.org/x/text v0.21.0 golang.org/x/tools v0.29.0 - google.golang.org/api v0.218.0 + google.golang.org/api v0.219.0 google.golang.org/genproto/googleapis/api v0.0.0-20250115164207-1a7da9e5054f google.golang.org/grpc v1.70.0 google.golang.org/protobuf v1.36.4 @@ -170,8 +170,8 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/morikuni/aec v1.0.0 // indirect - github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics v0.116.0 // indirect - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.116.0 // indirect + github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics v0.118.0 // indirect + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.118.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.2 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect @@ -196,7 +196,7 @@ require ( golang.org/x/net v0.34.0 // indirect golang.org/x/term v0.28.0 // indirect golang.org/x/time v0.9.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250124145028-65684f501c47 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/go.sum b/go.sum index 1e70e07ac2..5cf3e3ab67 100644 --- a/go.sum +++ b/go.sum @@ -87,14 +87,14 @@ github.com/dennwc/varint v1.0.0 h1:kGNFFSSw8ToIy3obO/kKr8U9GZYUAxQEVuix4zfDWzE= github.com/dennwc/varint v1.0.0/go.mod h1:hnItb35rvZvJrbTALZtY/iQfDs48JKRG1RPpgziApxA= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= -github.com/digitalocean/godo v1.132.0 h1:n0x6+ZkwbyQBtIU1wwBhv26EINqHg0wWQiBXlwYg/HQ= -github.com/digitalocean/godo v1.132.0/go.mod h1:PU8JB6I1XYkQIdHFop8lLAY9ojp6M0XcU0TWaQSxbrc= +github.com/digitalocean/godo v1.136.0 h1:DTxugljFJSMBPfEGq4KeXpnKeAHicggNqogcrw/YdZw= +github.com/digitalocean/godo v1.136.0/go.mod h1:PU8JB6I1XYkQIdHFop8lLAY9ojp6M0XcU0TWaQSxbrc= github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= -github.com/docker/docker v27.4.1+incompatible h1:ZJvcY7gfwHn1JF48PfbyXg7Jyt9ZCWDW+GGXOIxEwp4= -github.com/docker/docker v27.4.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v27.5.1+incompatible h1:4PYU5dnBYqRQi0294d1FBECqT9ECWeQAIfE8q4YnPY8= +github.com/docker/docker v27.5.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -273,8 +273,8 @@ github.com/hashicorp/nomad/api v0.0.0-20241218080744-e3ac00f30eec h1:+YBzb977Vrm github.com/hashicorp/nomad/api v0.0.0-20241218080744-e3ac00f30eec/go.mod h1:svtxn6QnrQ69P23VvIWMR34tg3vmwLz4UdUzm1dSCgE= github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY= github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4= -github.com/hetznercloud/hcloud-go/v2 v2.18.0 h1:BemrVGeWI8Kn/pvaC1jBsHZxQMnRqOydS7Ju4BERB4Q= -github.com/hetznercloud/hcloud-go/v2 v2.18.0/go.mod h1:r5RTzv+qi8IbLcDIskTzxkFIji7Ovc8yNgepQR9M+UA= +github.com/hetznercloud/hcloud-go/v2 v2.19.0 h1:crqbWMywudvlPLLczFf2hBpTPIATjrWMmwfiKSTpUt0= +github.com/hetznercloud/hcloud-go/v2 v2.19.0/go.mod h1:r5RTzv+qi8IbLcDIskTzxkFIji7Ovc8yNgepQR9M+UA= github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/ionos-cloud/sdk-go/v6 v6.3.2 h1:2mUmrZZz6cPyT9IRX0T8fBLc/7XU/eTxP2Y5tS7/09k= @@ -321,8 +321,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/linode/linodego v1.46.0 h1:+uOG4SD2MIrhbrLrvOD5HrbdLN3D19Wgn3MgdUNQjeU= -github.com/linode/linodego v1.46.0/go.mod h1:vyklQRzZUWhFVBZdYx4dcYJU/gG9yKB9VUcUs6ub0Lk= +github.com/linode/linodego v1.47.0 h1:6MFNCyzWbr8Rhl4r7d5DwZLwxvFIsM4ARH6W0KS/R0U= +github.com/linode/linodego v1.47.0/go.mod h1:vyklQRzZUWhFVBZdYx4dcYJU/gG9yKB9VUcUs6ub0Lk= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= @@ -391,14 +391,14 @@ github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= -github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics v0.116.0 h1:Kxk5Ral+Dc6VB9UmTketVjs+rbMZP8JxQ4SXDx4RivQ= -github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics v0.116.0/go.mod h1:ctT6oQmGmWGGGgUIKyx2fDwqz77N9+04gqKkDyAzKCg= -github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.116.0 h1:RlEK9MbxWyBHbLel8EJ1L7DbYVLai9dZL6Ljl2cBgyA= -github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.116.0/go.mod h1:AVUEyIjPb+0ARr7mhIkZkdNg3fd0ZcRhzAi53oZhl1Q= -github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.116.0 h1:jwnZYRBuPJnsKXE5H6ZvTEm91bXW5VP8+tLewzl54eg= -github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.116.0/go.mod h1:NT3Ag+DdnIAZQfD7l7OHwlYqnaAJ19SoPZ0nhD9yx4s= -github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor v0.116.0 h1:ZBmLuipJv7BT9fho/2yAFsS8AtMsCOCe4ON8oqkX3n8= -github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor v0.116.0/go.mod h1:f0GdYWGxUunyRZ088gHnoX78pc/gZc3dQlRtidiGXzg= +github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics v0.118.0 h1:PbknCwTbeTz8GNSfN4fOIp50YCDO19s1IAp6PGFcdpA= +github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics v0.118.0/go.mod h1:cOY+YDFtxJH3eQzJDObvWFFSIvD2AstG5MZ9t8wqusQ= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.118.0 h1:DSoYrOjLv23HXpx72hl61br4ZZTj6dqtwZSGoypKWIA= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.118.0/go.mod h1:nR+r7aAbsktscJk4fGmzljblbZBMaiZcIWeKbXV+HmY= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.118.0 h1:aUTSkzJExtrlHN32g8hX/cRNEo2ZmucPg+vwPqOYvhg= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.118.0/go.mod h1:a3sewj4nEozMwcNwZTHPzddS+1BnA6BaAkO/CRIGHVU= +github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor v0.118.0 h1:RZszYLp7sVMOD1rppjY+fP2PQh5qNAh5U6RoQNvd4Rg= +github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor v0.118.0/go.mod h1:5i928mwS+Ojv41l3/IxcyK1SCy6WnpL3wjLWKDb4YKQ= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= @@ -451,8 +451,8 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= -github.com/prometheus/sigv4 v0.1.1 h1:UJxjOqVcXctZlwDjpUpZ2OiMWJdFijgSofwLzO1Xk0Q= -github.com/prometheus/sigv4 v0.1.1/go.mod h1:RAmWVKqx0bwi0Qm4lrKMXFM0nhpesBcenfCtz9qRyH8= +github.com/prometheus/sigv4 v0.1.2 h1:R7570f8AoM5YnTUPFm3mjZH5q2k4D+I/phCWvZ4PXG8= +github.com/prometheus/sigv4 v0.1.2/go.mod h1:GF9fwrvLgkQwDdQ5BXeV9XUSCH/IPNqzvAoaohfjqMU= github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E= github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= @@ -511,8 +511,8 @@ go.opentelemetry.io/collector/component/componenttest v0.118.0 h1:knEHckoiL2fEWS go.opentelemetry.io/collector/component/componenttest v0.118.0/go.mod h1:aHc7t7zVwCpbhrWIWY+GMuaMxMCUP8C8P7pJOt8r/vU= go.opentelemetry.io/collector/config/configtelemetry v0.118.0 h1:UlN46EViG2X42odWtXgWaqY7Y01ZKpsnswSwXTWx5mM= go.opentelemetry.io/collector/config/configtelemetry v0.118.0/go.mod h1:SlBEwQg0qly75rXZ6W1Ig8jN25KBVBkFIIAUI1GiAAE= -go.opentelemetry.io/collector/confmap v1.22.0 h1:ZKQzRuj5lKu+seKArAAZ1yPRroDPricaIVIREm/jr3w= -go.opentelemetry.io/collector/confmap v1.22.0/go.mod h1:Rrhs+MWoaP6AswZp+ReQ2VO9dfOfcUjdjiSHBsG+nec= +go.opentelemetry.io/collector/confmap v1.24.0 h1:UUHVhkDCsVw14jPOarug9PDQE2vaB2ELPWMr7ARFBCA= +go.opentelemetry.io/collector/confmap v1.24.0/go.mod h1:Rrhs+MWoaP6AswZp+ReQ2VO9dfOfcUjdjiSHBsG+nec= go.opentelemetry.io/collector/consumer v1.24.0 h1:7DeyBm9qdr1EPuCfPjWyChPK16DbVc0wZeSa9LZprFU= go.opentelemetry.io/collector/consumer v1.24.0/go.mod h1:0G6jvZprIp4dpKMD1ZxCjriiP9GdFvFMObsQEtTk71s= go.opentelemetry.io/collector/consumer/consumertest v0.118.0 h1:8AAS9ejQapP1zqt0+cI6u+AUBheT3X0171N9WtXWsVY= @@ -670,8 +670,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.218.0 h1:x6JCjEWeZ9PFCRe9z0FBrNwj7pB7DOAqT35N+IPnAUA= -google.golang.org/api v0.218.0/go.mod h1:5VGHBAkxrA/8EFjLVEYmMUJ8/8+gWWQ3s4cFH0FxG2M= +google.golang.org/api v0.219.0 h1:nnKIvxKs/06jWawp2liznTBnMRQBEPpGo7I+oEypTX0= +google.golang.org/api v0.219.0/go.mod h1:K6OmjGm+NtLrIkHxv1U3a0qIf/0JOvAHd5O/6AoyKYE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -679,8 +679,8 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98 google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto/googleapis/api v0.0.0-20250115164207-1a7da9e5054f h1:gap6+3Gk41EItBuyi4XX/bp4oqJ3UwuIMl25yGinuAA= google.golang.org/genproto/googleapis/api v0.0.0-20250115164207-1a7da9e5054f/go.mod h1:Ic02D47M+zbarjYYUlK57y316f2MoN0gjAwI3f2S95o= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f h1:OxYkA3wjPsZyBylwymxSHa7ViiW1Sml4ToBrncvFehI= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f/go.mod h1:+2Yz8+CLJbIfL9z73EW45avw8Lmge3xVElCP9zEKi50= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250124145028-65684f501c47 h1:91mG8dNTpkC0uChJUQ9zCiRqx3GEEFOWaRZ0mI6Oj2I= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250124145028-65684f501c47/go.mod h1:+2Yz8+CLJbIfL9z73EW45avw8Lmge3xVElCP9zEKi50= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= diff --git a/model/textparse/benchmark_test.go b/model/textparse/benchmark_test.go index bd0d5089ac..4c5d850930 100644 --- a/model/textparse/benchmark_test.go +++ b/model/textparse/benchmark_test.go @@ -20,6 +20,7 @@ import ( "io" "os" "path/filepath" + "strings" "testing" "github.com/prometheus/prometheus/model/exemplar" @@ -30,32 +31,9 @@ import ( "github.com/stretchr/testify/require" ) -type newParser func([]byte, *labels.SymbolTable) Parser - -var newTestParserFns = map[string]newParser{ - "promtext": NewPromParser, - "promproto": func(b []byte, st *labels.SymbolTable) Parser { - return NewProtobufParser(b, true, st) - }, - "omtext": func(b []byte, st *labels.SymbolTable) Parser { - return NewOpenMetricsParser(b, st, WithOMParserCTSeriesSkipped()) - }, - "omtext_with_nhcb": func(b []byte, st *labels.SymbolTable) Parser { - p := NewOpenMetricsParser(b, st, WithOMParserCTSeriesSkipped()) - return NewNHCBParser(p, st, false) - }, -} - -// BenchmarkParse benchmarks parsing, mimicking how scrape/scrape.go#append use it. -// Typically used as follows: -/* - export bench=v1 && go test ./model/textparse/... \ - -run '^$' -bench '^BenchmarkParse' \ - -benchtime 2s -count 6 -cpu 2 -benchmem -timeout 999m \ - | tee ${bench}.txt -*/ -// For profiles, add -memprofile=${bench}.mem.pprof -cpuprofile=${bench}.cpu.pprof -// options. +// BenchmarkParse... set of benchmarks analyze efficiency of parsing various +// datasets with different parsers. It mimics how scrape/scrape.go#append use parsers +// and allows comparison with expfmt decoders if applicable. // // NOTE(bwplotka): Previous iterations of this benchmark had different cases for isolated // Series, Series+Metrics with and without reuse, Series+CT. Those cases are sometimes @@ -63,123 +41,224 @@ var newTestParserFns = map[string]newParser{ // make sense to persist such cases for everybody (e.g. for CI one day). // For local iteration, feel free to adjust cases/comment out code etc. // -// NOTE(bwplotka): Do not try to conclude "what parser (OM, proto, prom) is the fastest" -// as the testdata has different amount and type of metrics and features (e.g. exemplars). -func BenchmarkParse(b *testing.B) { - for _, bcase := range []struct { - dataFile string // Localized to "./testdata". - dataProto []byte - parser string +// NOTE(bwplotka): Those benchmarks are purposefully categorized per data-sets, +// to avoid temptation to assess "what parser (OM, proto, prom) is the fastest, +// in general" here due to not every parser supporting every data set type. +// Use scrape.BenchmarkScrapeLoopAppend if you want one benchmark comparing parsers fairly. - compareToExpfmtFormat expfmt.FormatType - }{ - {dataFile: "promtestdata.txt", parser: "promtext", compareToExpfmtFormat: expfmt.TypeTextPlain}, - {dataFile: "promtestdata.nometa.txt", parser: "promtext", compareToExpfmtFormat: expfmt.TypeTextPlain}, +/* + export bench=v1 && go test ./model/textparse/... \ + -run '^$' -bench '^BenchmarkParsePromText' \ + -benchtime 2s -count 6 -cpu 2 -benchmem -timeout 999m \ + | tee ${bench}.txt +*/ +func BenchmarkParsePromText(b *testing.B) { + data := readTestdataFile(b, "alltypes.237mfs.prom.txt") - // We don't pass compareToExpfmtFormat: expfmt.TypeProtoDelim as expfmt does not support GAUGE_HISTOGRAM, see https://github.com/prometheus/common/issues/430. - {dataProto: createTestProtoBuf(b).Bytes(), parser: "promproto"}, - - // We don't pass compareToExpfmtFormat: expfmt.TypeOpenMetrics as expfmt does not support OM exemplars, see https://github.com/prometheus/common/issues/703. - {dataFile: "omtestdata.txt", parser: "omtext"}, - {dataFile: "promtestdata.txt", parser: "omtext"}, // Compare how omtext parser deals with Prometheus text format vs promtext. - - // NHCB. - {dataFile: "omhistogramdata.txt", parser: "omtext"}, // Measure OM parser baseline for histograms. - {dataFile: "omhistogramdata.txt", parser: "omtext_with_nhcb"}, // Measure NHCB over OM parser. + for _, parser := range []string{ + "promtext", + "omtext", // Compare how omtext parser deals with Prometheus text format. + "expfmt-promtext", } { - var buf []byte - dataCase := bcase.dataFile - if len(bcase.dataProto) > 0 { - dataCase = "createTestProtoBuf()" - buf = bcase.dataProto - } else { - f, err := os.Open(filepath.Join("testdata", bcase.dataFile)) - require.NoError(b, err) - b.Cleanup(func() { - _ = f.Close() - }) - buf, err = io.ReadAll(f) - require.NoError(b, err) - } - b.Run(fmt.Sprintf("data=%v/parser=%v", dataCase, bcase.parser), func(b *testing.B) { - newParserFn := newTestParserFns[bcase.parser] - var ( - res labels.Labels - e exemplar.Exemplar - ) - - b.SetBytes(int64(len(buf))) - b.ReportAllocs() - b.ResetTimer() - - st := labels.NewSymbolTable() - for i := 0; i < b.N; i++ { - p := newParserFn(buf, st) - - Inner: - for { - t, err := p.Next() - switch t { - case EntryInvalid: - if errors.Is(err, io.EOF) { - break Inner - } - b.Fatal(err) - case EntryType: - _, _ = p.Type() - continue - case EntryHelp: - _, _ = p.Help() - continue - case EntryUnit: - _, _ = p.Unit() - continue - case EntryComment: - continue - case EntryHistogram: - _, _, _, _ = p.Histogram() - case EntrySeries: - _, _, _ = p.Series() - default: - b.Fatal("not implemented entry", t) - } - - _ = p.Metric(&res) - _ = p.CreatedTimestamp() - for hasExemplar := p.Exemplar(&e); hasExemplar; hasExemplar = p.Exemplar(&e) { - } - } - } - }) - - b.Run(fmt.Sprintf("data=%v/parser=xpfmt", dataCase), func(b *testing.B) { - if bcase.compareToExpfmtFormat == expfmt.TypeUnknown { - b.Skip("compareToExpfmtFormat not set") - } - - b.SetBytes(int64(len(buf))) - b.ReportAllocs() - b.ResetTimer() - - for i := 0; i < b.N; i++ { - decSamples := make(model.Vector, 0, 50) - sdec := expfmt.SampleDecoder{ - Dec: expfmt.NewDecoder(bytes.NewReader(buf), expfmt.NewFormat(bcase.compareToExpfmtFormat)), - Opts: &expfmt.DecodeOptions{ - Timestamp: model.TimeFromUnixNano(0), - }, - } - - for { - if err := sdec.Decode(&decSamples); err != nil { - if errors.Is(err, io.EOF) { - break - } - b.Fatal(err) - } - decSamples = decSamples[:0] - } + b.Run(fmt.Sprintf("parser=%v", parser), func(b *testing.B) { + if strings.HasPrefix(parser, "expfmt-") { + benchExpFmt(b, data, parser) + } else { + benchParse(b, data, parser) } }) } } + +/* + export bench=v1 && go test ./model/textparse/... \ + -run '^$' -bench '^BenchmarkParsePromText_NoMeta' \ + -benchtime 2s -count 6 -cpu 2 -benchmem -timeout 999m \ + | tee ${bench}.txt +*/ +func BenchmarkParsePromText_NoMeta(b *testing.B) { + data := readTestdataFile(b, "alltypes.237mfs.nometa.prom.txt") + + for _, parser := range []string{ + "promtext", + "expfmt-promtext", + } { + b.Run(fmt.Sprintf("parser=%v", parser), func(b *testing.B) { + if strings.HasPrefix(parser, "expfmt-") { + benchExpFmt(b, data, parser) + } else { + benchParse(b, data, parser) + } + }) + } +} + +/* + export bench=v1 && go test ./model/textparse/... \ + -run '^$' -bench '^BenchmarkParseOMText' \ + -benchtime 2s -count 6 -cpu 2 -benchmem -timeout 999m \ + | tee ${bench}.txt +*/ +func BenchmarkParseOMText(b *testing.B) { + data := readTestdataFile(b, "alltypes.5mfs.om.txt") + // TODO(bwplotka): Add comparison with expfmt.TypeOpenMetrics once expfmt + // support OM exemplars, see https://github.com/prometheus/common/issues/703. + benchParse(b, data, "omtext") +} + +/* + export bench=v1 && go test ./model/textparse/... \ + -run '^$' -bench '^BenchmarkParsePromProto' \ + -benchtime 2s -count 6 -cpu 2 -benchmem -timeout 999m \ + | tee ${bench}.txt +*/ +func BenchmarkParsePromProto(b *testing.B) { + data := createTestProtoBuf(b).Bytes() + // TODO(bwplotka): Add comparison with expfmt.TypeProtoDelim once expfmt + // support GAUGE_HISTOGRAM, see https://github.com/prometheus/common/issues/430. + benchParse(b, data, "promproto") +} + +/* + export bench=v1 && go test ./model/textparse/... \ + -run '^$' -bench '^BenchmarkParseOpenMetricsNHCB' \ + -benchtime 2s -count 6 -cpu 2 -benchmem -timeout 999m \ + | tee ${bench}.txt +*/ +func BenchmarkParseOpenMetricsNHCB(b *testing.B) { + data := readTestdataFile(b, "1histogram.om.txt") + + for _, parser := range []string{ + "omtext", // Measure OM parser baseline for histograms. + "omtext_with_nhcb", // Measure NHCB over OM parser. + } { + b.Run(fmt.Sprintf("parser=%v", parser), func(b *testing.B) { + benchParse(b, data, parser) + }) + } +} + +func benchParse(b *testing.B, data []byte, parser string) { + type newParser func([]byte, *labels.SymbolTable) Parser + + var newParserFn newParser + switch parser { + case "promtext": + newParserFn = NewPromParser + case "promproto": + newParserFn = func(b []byte, st *labels.SymbolTable) Parser { + return NewProtobufParser(b, true, st) + } + case "omtext": + newParserFn = func(b []byte, st *labels.SymbolTable) Parser { + return NewOpenMetricsParser(b, st, WithOMParserCTSeriesSkipped()) + } + case "omtext_with_nhcb": + newParserFn = func(b []byte, st *labels.SymbolTable) Parser { + p := NewOpenMetricsParser(b, st, WithOMParserCTSeriesSkipped()) + return NewNHCBParser(p, st, false) + } + default: + b.Fatal("unknown parser", parser) + } + + var ( + res labels.Labels + e exemplar.Exemplar + ) + + b.SetBytes(int64(len(data))) + b.ReportAllocs() + b.ResetTimer() + + st := labels.NewSymbolTable() + for i := 0; i < b.N; i++ { + p := newParserFn(data, st) + + Inner: + for { + t, err := p.Next() + switch t { + case EntryInvalid: + if errors.Is(err, io.EOF) { + break Inner + } + b.Fatal(err) + case EntryType: + _, _ = p.Type() + continue + case EntryHelp: + _, _ = p.Help() + continue + case EntryUnit: + _, _ = p.Unit() + continue + case EntryComment: + continue + case EntryHistogram: + _, _, _, _ = p.Histogram() + case EntrySeries: + _, _, _ = p.Series() + default: + b.Fatal("not implemented entry", t) + } + + _ = p.Metric(&res) + _ = p.CreatedTimestamp() + for hasExemplar := p.Exemplar(&e); hasExemplar; hasExemplar = p.Exemplar(&e) { + } + } + } +} + +func benchExpFmt(b *testing.B, data []byte, expFormatTypeStr string) { + expfmtFormatType := expfmt.TypeUnknown + switch expFormatTypeStr { + case "expfmt-promtext": + expfmtFormatType = expfmt.TypeProtoText + case "expfmt-promproto": + expfmtFormatType = expfmt.TypeProtoDelim + case "expfmt-omtext": + expfmtFormatType = expfmt.TypeOpenMetrics + default: + b.Fatal("unknown expfmt format type", expFormatTypeStr) + } + + b.SetBytes(int64(len(data))) + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + decSamples := make(model.Vector, 0, 50) + sdec := expfmt.SampleDecoder{ + Dec: expfmt.NewDecoder(bytes.NewReader(data), expfmt.NewFormat(expfmtFormatType)), + Opts: &expfmt.DecodeOptions{ + Timestamp: model.TimeFromUnixNano(0), + }, + } + + for { + if err := sdec.Decode(&decSamples); err != nil { + if errors.Is(err, io.EOF) { + break + } + b.Fatal(err) + } + decSamples = decSamples[:0] + } + } +} + +func readTestdataFile(tb testing.TB, file string) []byte { + tb.Helper() + + f, err := os.Open(filepath.Join("testdata", file)) + require.NoError(tb, err) + + tb.Cleanup(func() { + _ = f.Close() + }) + buf, err := io.ReadAll(f) + require.NoError(tb, err) + return buf +} diff --git a/model/textparse/testdata/omhistogramdata.txt b/model/textparse/testdata/1histogram.om.txt similarity index 100% rename from model/textparse/testdata/omhistogramdata.txt rename to model/textparse/testdata/1histogram.om.txt diff --git a/model/textparse/testdata/alltypes.237mfs.nometa.prom.txt b/model/textparse/testdata/alltypes.237mfs.nometa.prom.txt new file mode 100644 index 0000000000..8724a7b837 --- /dev/null +++ b/model/textparse/testdata/alltypes.237mfs.nometa.prom.txt @@ -0,0 +1,1858 @@ +go_gc_cycles_automatic_gc_cycles_total 190932 +go_gc_cycles_forced_gc_cycles_total 0 +go_gc_cycles_total_gc_cycles_total 190932 +go_gc_duration_seconds{quantile="0"} 7.6238e-05 +go_gc_duration_seconds{quantile="0.25"} 0.00010188 +go_gc_duration_seconds{quantile="0.5"} 0.000135819 +go_gc_duration_seconds{quantile="0.75"} 0.000156061 +go_gc_duration_seconds{quantile="1"} 0.002389378 +go_gc_duration_seconds_sum 44.985611511 +go_gc_duration_seconds_count 190932 +go_gc_gogc_percent 75 +go_gc_gomemlimit_bytes 9.03676723e+08 +go_gc_heap_allocs_by_size_bytes_bucket{le="8.999999999999998"} 2.279966416e+09 +go_gc_heap_allocs_by_size_bytes_bucket{le="24.999999999999996"} 1.9429106442e+10 +go_gc_heap_allocs_by_size_bytes_bucket{le="64.99999999999999"} 3.2158220609e+10 +go_gc_heap_allocs_by_size_bytes_bucket{le="144.99999999999997"} 4.2309744198e+10 +go_gc_heap_allocs_by_size_bytes_bucket{le="320.99999999999994"} 4.3418919481e+10 +go_gc_heap_allocs_by_size_bytes_bucket{le="704.9999999999999"} 4.374631622e+10 +go_gc_heap_allocs_by_size_bytes_bucket{le="1536.9999999999998"} 4.3917578245e+10 +go_gc_heap_allocs_by_size_bytes_bucket{le="3200.9999999999995"} 4.396605609e+10 +go_gc_heap_allocs_by_size_bytes_bucket{le="6528.999999999999"} 4.4007501305e+10 +go_gc_heap_allocs_by_size_bytes_bucket{le="13568.999999999998"} 4.4020325917e+10 +go_gc_heap_allocs_by_size_bytes_bucket{le="27264.999999999996"} 4.4036187548e+10 +go_gc_heap_allocs_by_size_bytes_bucket{le="+Inf"} 4.4046288803e+10 +go_gc_heap_allocs_by_size_bytes_sum 5.937322571024e+12 +go_gc_heap_allocs_by_size_bytes_count 4.4046288803e+10 +go_gc_heap_allocs_bytes_total 5.937322571024e+12 +go_gc_heap_allocs_objects_total 4.4046288803e+10 +go_gc_heap_frees_by_size_bytes_bucket{le="8.999999999999998"} 2.279951045e+09 +go_gc_heap_frees_by_size_bytes_bucket{le="24.999999999999996"} 1.9428965693e+10 +go_gc_heap_frees_by_size_bytes_bucket{le="64.99999999999999"} 3.2157849717e+10 +go_gc_heap_frees_by_size_bytes_bucket{le="144.99999999999997"} 4.2309204178e+10 +go_gc_heap_frees_by_size_bytes_bucket{le="320.99999999999994"} 4.3418348856e+10 +go_gc_heap_frees_by_size_bytes_bucket{le="704.9999999999999"} 4.3745739652e+10 +go_gc_heap_frees_by_size_bytes_bucket{le="1536.9999999999998"} 4.3916999773e+10 +go_gc_heap_frees_by_size_bytes_bucket{le="3200.9999999999995"} 4.3965477112e+10 +go_gc_heap_frees_by_size_bytes_bucket{le="6528.999999999999"} 4.4006921621e+10 +go_gc_heap_frees_by_size_bytes_bucket{le="13568.999999999998"} 4.4019746017e+10 +go_gc_heap_frees_by_size_bytes_bucket{le="27264.999999999996"} 4.4035607466e+10 +go_gc_heap_frees_by_size_bytes_bucket{le="+Inf"} 4.4045708586e+10 +go_gc_heap_frees_by_size_bytes_sum 5.937239047736e+12 +go_gc_heap_frees_by_size_bytes_count 4.4045708586e+10 +go_gc_heap_frees_bytes_total 5.937239047736e+12 +go_gc_heap_frees_objects_total 4.4045708586e+10 +go_gc_heap_goal_bytes 1.0365948e+08 +go_gc_heap_live_bytes 5.8791024e+07 +go_gc_heap_objects_objects 580217 +go_gc_heap_tiny_allocs_objects_total 8.306064403e+09 +go_gc_limiter_last_enabled_gc_cycle 160896 +go_gc_pauses_seconds_bucket{le="6.399999999999999e-08"} 0 +go_gc_pauses_seconds_bucket{le="6.399999999999999e-07"} 0 +go_gc_pauses_seconds_bucket{le="7.167999999999999e-06"} 137212 +go_gc_pauses_seconds_bucket{le="8.191999999999999e-05"} 208425 +go_gc_pauses_seconds_bucket{le="0.0009175039999999999"} 376121 +go_gc_pauses_seconds_bucket{le="0.010485759999999998"} 381798 +go_gc_pauses_seconds_bucket{le="0.11744051199999998"} 381863 +go_gc_pauses_seconds_bucket{le="+Inf"} 381864 +go_gc_pauses_seconds_sum 20.343611904000003 +go_gc_pauses_seconds_count 381864 +go_gc_scan_globals_bytes 555824 +go_gc_scan_heap_bytes 4.0986192e+07 +go_gc_scan_stack_bytes 477760 +go_gc_scan_total_bytes 4.2019776e+07 +go_gc_stack_starting_size_bytes 4096 +go_goroutines 151 +go_info{version="go1.23.4"} 1 +go_memstats_alloc_bytes 8.3523288e+07 +go_memstats_alloc_bytes_total 5.937322571024e+12 +go_memstats_buck_hash_sys_bytes 9.35076e+06 +go_memstats_frees_total 5.2351772989e+10 +go_memstats_gc_sys_bytes 5.450544e+06 +go_memstats_heap_alloc_bytes 8.3523288e+07 +go_memstats_heap_idle_bytes 1.23691008e+08 +go_memstats_heap_inuse_bytes 9.56416e+07 +go_memstats_heap_objects 580217 +go_memstats_heap_released_bytes 1.05897984e+08 +go_memstats_heap_sys_bytes 2.19332608e+08 +go_memstats_last_gc_time_seconds 1.738949372347471e+09 +go_memstats_mallocs_total 5.2352353206e+10 +go_memstats_mcache_inuse_bytes 1200 +go_memstats_mcache_sys_bytes 15600 +go_memstats_mspan_inuse_bytes 1.21232e+06 +go_memstats_mspan_sys_bytes 2.13792e+06 +go_memstats_next_gc_bytes 1.0365948e+08 +go_memstats_other_sys_bytes 803696 +go_memstats_stack_inuse_bytes 2.818048e+06 +go_memstats_stack_sys_bytes 2.818048e+06 +go_memstats_sys_bytes 2.39909176e+08 +go_sched_gomaxprocs_threads 1 +go_sched_goroutines_goroutines 151 +go_sched_latencies_seconds_bucket{le="6.399999999999999e-08"} 5.0101035e+07 +go_sched_latencies_seconds_bucket{le="6.399999999999999e-07"} 7.4291762e+07 +go_sched_latencies_seconds_bucket{le="7.167999999999999e-06"} 8.1386236e+07 +go_sched_latencies_seconds_bucket{le="8.191999999999999e-05"} 8.2116161e+07 +go_sched_latencies_seconds_bucket{le="0.0009175039999999999"} 8.2802009e+07 +go_sched_latencies_seconds_bucket{le="0.010485759999999998"} 8.6274195e+07 +go_sched_latencies_seconds_bucket{le="0.11744051199999998"} 8.6724074e+07 +go_sched_latencies_seconds_bucket{le="+Inf"} 8.6726596e+07 +go_sched_latencies_seconds_sum 8266.758178496 +go_sched_latencies_seconds_count 8.6726596e+07 +go_sched_pauses_stopping_gc_seconds_bucket{le="6.399999999999999e-08"} 0 +go_sched_pauses_stopping_gc_seconds_bucket{le="6.399999999999999e-07"} 236272 +go_sched_pauses_stopping_gc_seconds_bucket{le="7.167999999999999e-06"} 380799 +go_sched_pauses_stopping_gc_seconds_bucket{le="8.191999999999999e-05"} 381741 +go_sched_pauses_stopping_gc_seconds_bucket{le="0.0009175039999999999"} 381807 +go_sched_pauses_stopping_gc_seconds_bucket{le="0.010485759999999998"} 381862 +go_sched_pauses_stopping_gc_seconds_bucket{le="0.11744051199999998"} 381864 +go_sched_pauses_stopping_gc_seconds_bucket{le="+Inf"} 381864 +go_sched_pauses_stopping_gc_seconds_sum 0.191211904 +go_sched_pauses_stopping_gc_seconds_count 381864 +go_sched_pauses_stopping_other_seconds_bucket{le="6.399999999999999e-08"} 0 +go_sched_pauses_stopping_other_seconds_bucket{le="6.399999999999999e-07"} 0 +go_sched_pauses_stopping_other_seconds_bucket{le="7.167999999999999e-06"} 0 +go_sched_pauses_stopping_other_seconds_bucket{le="8.191999999999999e-05"} 0 +go_sched_pauses_stopping_other_seconds_bucket{le="0.0009175039999999999"} 0 +go_sched_pauses_stopping_other_seconds_bucket{le="0.010485759999999998"} 0 +go_sched_pauses_stopping_other_seconds_bucket{le="0.11744051199999998"} 0 +go_sched_pauses_stopping_other_seconds_bucket{le="+Inf"} 0 +go_sched_pauses_stopping_other_seconds_sum 0 +go_sched_pauses_stopping_other_seconds_count 0 +go_sched_pauses_total_gc_seconds_bucket{le="6.399999999999999e-08"} 0 +go_sched_pauses_total_gc_seconds_bucket{le="6.399999999999999e-07"} 0 +go_sched_pauses_total_gc_seconds_bucket{le="7.167999999999999e-06"} 137212 +go_sched_pauses_total_gc_seconds_bucket{le="8.191999999999999e-05"} 208425 +go_sched_pauses_total_gc_seconds_bucket{le="0.0009175039999999999"} 376121 +go_sched_pauses_total_gc_seconds_bucket{le="0.010485759999999998"} 381798 +go_sched_pauses_total_gc_seconds_bucket{le="0.11744051199999998"} 381863 +go_sched_pauses_total_gc_seconds_bucket{le="+Inf"} 381864 +go_sched_pauses_total_gc_seconds_sum 20.343611904000003 +go_sched_pauses_total_gc_seconds_count 381864 +go_sched_pauses_total_other_seconds_bucket{le="6.399999999999999e-08"} 0 +go_sched_pauses_total_other_seconds_bucket{le="6.399999999999999e-07"} 0 +go_sched_pauses_total_other_seconds_bucket{le="7.167999999999999e-06"} 0 +go_sched_pauses_total_other_seconds_bucket{le="8.191999999999999e-05"} 0 +go_sched_pauses_total_other_seconds_bucket{le="0.0009175039999999999"} 0 +go_sched_pauses_total_other_seconds_bucket{le="0.010485759999999998"} 0 +go_sched_pauses_total_other_seconds_bucket{le="0.11744051199999998"} 0 +go_sched_pauses_total_other_seconds_bucket{le="+Inf"} 0 +go_sched_pauses_total_other_seconds_sum 0 +go_sched_pauses_total_other_seconds_count 0 +go_sync_mutex_wait_total_seconds_total 628.29966272 +go_threads 10 +net_conntrack_dialer_conn_attempted_total{dialer_name="alertmanager"} 2 +net_conntrack_dialer_conn_attempted_total{dialer_name="blackbox"} 1 +net_conntrack_dialer_conn_attempted_total{dialer_name="caddy"} 1 +net_conntrack_dialer_conn_attempted_total{dialer_name="cadvisor"} 2 +net_conntrack_dialer_conn_attempted_total{dialer_name="default"} 0 +net_conntrack_dialer_conn_attempted_total{dialer_name="grafana"} 7 +net_conntrack_dialer_conn_attempted_total{dialer_name="node"} 5 +net_conntrack_dialer_conn_attempted_total{dialer_name="prometheus"} 1 +net_conntrack_dialer_conn_attempted_total{dialer_name="random"} 4 +net_conntrack_dialer_conn_closed_total{dialer_name="alertmanager"} 0 +net_conntrack_dialer_conn_closed_total{dialer_name="blackbox"} 0 +net_conntrack_dialer_conn_closed_total{dialer_name="caddy"} 0 +net_conntrack_dialer_conn_closed_total{dialer_name="cadvisor"} 1 +net_conntrack_dialer_conn_closed_total{dialer_name="default"} 0 +net_conntrack_dialer_conn_closed_total{dialer_name="grafana"} 4 +net_conntrack_dialer_conn_closed_total{dialer_name="node"} 4 +net_conntrack_dialer_conn_closed_total{dialer_name="prometheus"} 0 +net_conntrack_dialer_conn_closed_total{dialer_name="random"} 0 +net_conntrack_dialer_conn_established_total{dialer_name="alertmanager"} 2 +net_conntrack_dialer_conn_established_total{dialer_name="blackbox"} 1 +net_conntrack_dialer_conn_established_total{dialer_name="caddy"} 1 +net_conntrack_dialer_conn_established_total{dialer_name="cadvisor"} 2 +net_conntrack_dialer_conn_established_total{dialer_name="default"} 0 +net_conntrack_dialer_conn_established_total{dialer_name="grafana"} 5 +net_conntrack_dialer_conn_established_total{dialer_name="node"} 5 +net_conntrack_dialer_conn_established_total{dialer_name="prometheus"} 1 +net_conntrack_dialer_conn_established_total{dialer_name="random"} 4 +net_conntrack_dialer_conn_failed_total{dialer_name="alertmanager",reason="refused"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="alertmanager",reason="resolution"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="alertmanager",reason="timeout"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="alertmanager",reason="unknown"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="blackbox",reason="refused"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="blackbox",reason="resolution"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="blackbox",reason="timeout"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="blackbox",reason="unknown"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="caddy",reason="refused"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="caddy",reason="resolution"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="caddy",reason="timeout"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="caddy",reason="unknown"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="cadvisor",reason="refused"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="cadvisor",reason="resolution"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="cadvisor",reason="timeout"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="cadvisor",reason="unknown"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="default",reason="refused"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="default",reason="resolution"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="default",reason="timeout"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="default",reason="unknown"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="grafana",reason="refused"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="grafana",reason="resolution"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="grafana",reason="timeout"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="grafana",reason="unknown"} 2 +net_conntrack_dialer_conn_failed_total{dialer_name="node",reason="refused"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="node",reason="resolution"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="node",reason="timeout"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="node",reason="unknown"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="prometheus",reason="refused"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="prometheus",reason="resolution"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="prometheus",reason="timeout"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="prometheus",reason="unknown"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="random",reason="refused"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="random",reason="resolution"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="random",reason="timeout"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="random",reason="unknown"} 0 +net_conntrack_listener_conn_accepted_total{listener_name="http"} 561771 +net_conntrack_listener_conn_closed_total{listener_name="http"} 561723 +process_cpu_seconds_total 96501.42 +process_max_fds 65000 +process_network_receive_bytes_total 9.89686846606e+11 +process_network_transmit_bytes_total 3.743928187168e+12 +process_open_fds 124 +process_resident_memory_bytes 1.72822528e+08 +process_start_time_seconds 1.73806369023e+09 +process_virtual_memory_bytes 4.608667648e+09 +process_virtual_memory_max_bytes 1.8446744073709552e+19 +prometheus_api_notification_active_subscribers 2 +prometheus_api_notification_updates_dropped_total 0 +prometheus_api_notification_updates_sent_total 5 +prometheus_api_remote_read_queries 0 +prometheus_build_info{branch="HEAD",goarch="amd64",goos="linux",goversion="go1.23.4",revision="7086161a93b262aa0949dbf2aba15a5a7b13e0a3",tags="netgo,builtinassets,stringlabels",version="3.1.0"} 1 +prometheus_config_last_reload_success_timestamp_seconds 1.7380636976181264e+09 +prometheus_config_last_reload_successful 1 +prometheus_engine_queries 0 +prometheus_engine_queries_concurrent_max 20 +prometheus_engine_query_duration_seconds{slice="inner_eval",quantile="0.5"} 8.075e-05 +prometheus_engine_query_duration_seconds{slice="inner_eval",quantile="0.9"} 0.000917449 +prometheus_engine_query_duration_seconds{slice="inner_eval",quantile="0.99"} 0.009315769 +prometheus_engine_query_duration_seconds_sum{slice="inner_eval"} 12506.67007997419 +prometheus_engine_query_duration_seconds_count{slice="inner_eval"} 8.714071e+06 +prometheus_engine_query_duration_seconds{slice="prepare_time",quantile="0.5"} 2.228e-05 +prometheus_engine_query_duration_seconds{slice="prepare_time",quantile="0.9"} 6.2819e-05 +prometheus_engine_query_duration_seconds{slice="prepare_time",quantile="0.99"} 0.000399637 +prometheus_engine_query_duration_seconds_sum{slice="prepare_time"} 490.326247638047 +prometheus_engine_query_duration_seconds_count{slice="prepare_time"} 8.714071e+06 +prometheus_engine_query_duration_seconds{slice="queue_time",quantile="0.5"} 4.628e-06 +prometheus_engine_query_duration_seconds{slice="queue_time",quantile="0.9"} 1.6082e-05 +prometheus_engine_query_duration_seconds{slice="queue_time",quantile="0.99"} 4.1174e-05 +prometheus_engine_query_duration_seconds_sum{slice="queue_time"} 8720.662071224393 +prometheus_engine_query_duration_seconds_count{slice="queue_time"} 1.7428526e+07 +prometheus_engine_query_duration_seconds{slice="result_sort",quantile="0.5"} 7.83e-07 +prometheus_engine_query_duration_seconds{slice="result_sort",quantile="0.9"} 1.994e-06 +prometheus_engine_query_duration_seconds{slice="result_sort",quantile="0.99"} 1.458e-05 +prometheus_engine_query_duration_seconds_sum{slice="result_sort"} 2.337879348999977 +prometheus_engine_query_duration_seconds_count{slice="result_sort"} 1.208154e+06 +prometheus_engine_query_log_enabled 0 +prometheus_engine_query_log_failures_total 0 +prometheus_engine_query_samples_total 9.1183747239e+10 +prometheus_http_request_duration_seconds_bucket{handler="/",le="0.1"} 688 +prometheus_http_request_duration_seconds_bucket{handler="/",le="0.2"} 688 +prometheus_http_request_duration_seconds_bucket{handler="/",le="0.4"} 688 +prometheus_http_request_duration_seconds_bucket{handler="/",le="1"} 688 +prometheus_http_request_duration_seconds_bucket{handler="/",le="3"} 688 +prometheus_http_request_duration_seconds_bucket{handler="/",le="8"} 688 +prometheus_http_request_duration_seconds_bucket{handler="/",le="20"} 688 +prometheus_http_request_duration_seconds_bucket{handler="/",le="60"} 688 +prometheus_http_request_duration_seconds_bucket{handler="/",le="120"} 688 +prometheus_http_request_duration_seconds_bucket{handler="/",le="+Inf"} 688 +prometheus_http_request_duration_seconds_sum{handler="/"} 0.026826932000000022 +prometheus_http_request_duration_seconds_count{handler="/"} 688 +prometheus_http_request_duration_seconds_bucket{handler="/-/healthy",le="0.1"} 29524 +prometheus_http_request_duration_seconds_bucket{handler="/-/healthy",le="0.2"} 29524 +prometheus_http_request_duration_seconds_bucket{handler="/-/healthy",le="0.4"} 29524 +prometheus_http_request_duration_seconds_bucket{handler="/-/healthy",le="1"} 29524 +prometheus_http_request_duration_seconds_bucket{handler="/-/healthy",le="3"} 29524 +prometheus_http_request_duration_seconds_bucket{handler="/-/healthy",le="8"} 29524 +prometheus_http_request_duration_seconds_bucket{handler="/-/healthy",le="20"} 29524 +prometheus_http_request_duration_seconds_bucket{handler="/-/healthy",le="60"} 29524 +prometheus_http_request_duration_seconds_bucket{handler="/-/healthy",le="120"} 29524 +prometheus_http_request_duration_seconds_bucket{handler="/-/healthy",le="+Inf"} 29524 +prometheus_http_request_duration_seconds_sum{handler="/-/healthy"} 0.7400570460000002 +prometheus_http_request_duration_seconds_count{handler="/-/healthy"} 29524 +prometheus_http_request_duration_seconds_bucket{handler="/-/ready",le="0.1"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/-/ready",le="0.2"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/-/ready",le="0.4"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/-/ready",le="1"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/-/ready",le="3"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/-/ready",le="8"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/-/ready",le="20"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/-/ready",le="60"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/-/ready",le="120"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/-/ready",le="+Inf"} 49 +prometheus_http_request_duration_seconds_sum{handler="/-/ready"} 0.005040123000000002 +prometheus_http_request_duration_seconds_count{handler="/-/ready"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/alerts",le="0.1"} 48 +prometheus_http_request_duration_seconds_bucket{handler="/alerts",le="0.2"} 48 +prometheus_http_request_duration_seconds_bucket{handler="/alerts",le="0.4"} 48 +prometheus_http_request_duration_seconds_bucket{handler="/alerts",le="1"} 48 +prometheus_http_request_duration_seconds_bucket{handler="/alerts",le="3"} 48 +prometheus_http_request_duration_seconds_bucket{handler="/alerts",le="8"} 48 +prometheus_http_request_duration_seconds_bucket{handler="/alerts",le="20"} 48 +prometheus_http_request_duration_seconds_bucket{handler="/alerts",le="60"} 48 +prometheus_http_request_duration_seconds_bucket{handler="/alerts",le="120"} 48 +prometheus_http_request_duration_seconds_bucket{handler="/alerts",le="+Inf"} 48 +prometheus_http_request_duration_seconds_sum{handler="/alerts"} 0.011801602999999999 +prometheus_http_request_duration_seconds_count{handler="/alerts"} 48 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/*path",le="0.1"} 27 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/*path",le="0.2"} 27 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/*path",le="0.4"} 27 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/*path",le="1"} 27 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/*path",le="3"} 27 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/*path",le="8"} 27 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/*path",le="20"} 27 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/*path",le="60"} 27 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/*path",le="120"} 27 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/*path",le="+Inf"} 27 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/*path"} 0.001724389 +prometheus_http_request_duration_seconds_count{handler="/api/v1/*path"} 27 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/alertmanagers",le="0.1"} 8 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/alertmanagers",le="0.2"} 8 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/alertmanagers",le="0.4"} 8 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/alertmanagers",le="1"} 8 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/alertmanagers",le="3"} 8 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/alertmanagers",le="8"} 8 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/alertmanagers",le="20"} 8 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/alertmanagers",le="60"} 8 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/alertmanagers",le="120"} 8 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/alertmanagers",le="+Inf"} 8 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/alertmanagers"} 0.042492975999999995 +prometheus_http_request_duration_seconds_count{handler="/api/v1/alertmanagers"} 8 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/alerts",le="0.1"} 14630 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/alerts",le="0.2"} 14635 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/alerts",le="0.4"} 14635 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/alerts",le="1"} 14635 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/alerts",le="3"} 14635 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/alerts",le="8"} 14635 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/alerts",le="20"} 14635 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/alerts",le="60"} 14635 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/alerts",le="120"} 14635 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/alerts",le="+Inf"} 14635 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/alerts"} 19.028669391999912 +prometheus_http_request_duration_seconds_count{handler="/api/v1/alerts"} 14635 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/format_query",le="0.1"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/format_query",le="0.2"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/format_query",le="0.4"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/format_query",le="1"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/format_query",le="3"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/format_query",le="8"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/format_query",le="20"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/format_query",le="60"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/format_query",le="120"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/format_query",le="+Inf"} 4 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/format_query"} 0.023786675 +prometheus_http_request_duration_seconds_count{handler="/api/v1/format_query"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/label/:name/values",le="0.1"} 17773 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/label/:name/values",le="0.2"} 17860 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/label/:name/values",le="0.4"} 17939 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/label/:name/values",le="1"} 17970 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/label/:name/values",le="3"} 17971 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/label/:name/values",le="8"} 17971 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/label/:name/values",le="20"} 17971 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/label/:name/values",le="60"} 17971 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/label/:name/values",le="120"} 17971 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/label/:name/values",le="+Inf"} 17971 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/label/:name/values"} 99.95326355699925 +prometheus_http_request_duration_seconds_count{handler="/api/v1/label/:name/values"} 17971 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/labels",le="0.1"} 4905 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/labels",le="0.2"} 4912 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/labels",le="0.4"} 4916 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/labels",le="1"} 4921 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/labels",le="3"} 4921 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/labels",le="8"} 4921 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/labels",le="20"} 4921 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/labels",le="60"} 4921 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/labels",le="120"} 4921 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/labels",le="+Inf"} 4921 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/labels"} 12.963155722000025 +prometheus_http_request_duration_seconds_count{handler="/api/v1/labels"} 4921 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/metadata",le="0.1"} 2073 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/metadata",le="0.2"} 2091 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/metadata",le="0.4"} 2093 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/metadata",le="1"} 2093 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/metadata",le="3"} 2093 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/metadata",le="8"} 2093 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/metadata",le="20"} 2093 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/metadata",le="60"} 2093 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/metadata",le="120"} 2093 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/metadata",le="+Inf"} 2093 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/metadata"} 16.90794258600001 +prometheus_http_request_duration_seconds_count{handler="/api/v1/metadata"} 2093 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/notifications",le="0.1"} 12 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/notifications",le="0.2"} 12 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/notifications",le="0.4"} 12 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/notifications",le="1"} 12 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/notifications",le="3"} 12 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/notifications",le="8"} 12 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/notifications",le="20"} 12 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/notifications",le="60"} 12 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/notifications",le="120"} 12 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/notifications",le="+Inf"} 12 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/notifications"} 0.029363403 +prometheus_http_request_duration_seconds_count{handler="/api/v1/notifications"} 12 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/notifications/live",le="0.1"} 3 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/notifications/live",le="0.2"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/notifications/live",le="0.4"} 11 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/notifications/live",le="1"} 29 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/notifications/live",le="3"} 79 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/notifications/live",le="8"} 116 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/notifications/live",le="20"} 187 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/notifications/live",le="60"} 224 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/notifications/live",le="120"} 234 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/notifications/live",le="+Inf"} 263 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/notifications/live"} 85150.78366682903 +prometheus_http_request_duration_seconds_count{handler="/api/v1/notifications/live"} 263 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/parse_query",le="0.1"} 265 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/parse_query",le="0.2"} 265 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/parse_query",le="0.4"} 265 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/parse_query",le="1"} 265 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/parse_query",le="3"} 265 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/parse_query",le="8"} 265 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/parse_query",le="20"} 265 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/parse_query",le="60"} 265 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/parse_query",le="120"} 265 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/parse_query",le="+Inf"} 265 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/parse_query"} 0.7265021480000003 +prometheus_http_request_duration_seconds_count{handler="/api/v1/parse_query"} 265 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query",le="0.1"} 3051 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query",le="0.2"} 3082 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query",le="0.4"} 3173 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query",le="1"} 3213 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query",le="3"} 3216 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query",le="8"} 3217 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query",le="20"} 3217 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query",le="60"} 3217 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query",le="120"} 3217 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query",le="+Inf"} 3217 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/query"} 70.23571750499987 +prometheus_http_request_duration_seconds_count{handler="/api/v1/query"} 3217 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query_exemplars",le="0.1"} 831 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query_exemplars",le="0.2"} 839 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query_exemplars",le="0.4"} 841 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query_exemplars",le="1"} 841 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query_exemplars",le="3"} 841 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query_exemplars",le="8"} 841 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query_exemplars",le="20"} 841 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query_exemplars",le="60"} 841 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query_exemplars",le="120"} 841 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query_exemplars",le="+Inf"} 841 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/query_exemplars"} 3.6569132899999968 +prometheus_http_request_duration_seconds_count{handler="/api/v1/query_exemplars"} 841 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query_range",le="0.1"} 116338 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query_range",le="0.2"} 116784 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query_range",le="0.4"} 117924 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query_range",le="1"} 118147 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query_range",le="3"} 118155 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query_range",le="8"} 118157 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query_range",le="20"} 118157 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query_range",le="60"} 118157 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query_range",le="120"} 118157 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query_range",le="+Inf"} 118157 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/query_range"} 832.1240439999854 +prometheus_http_request_duration_seconds_count{handler="/api/v1/query_range"} 118157 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/read",le="0.1"} 16 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/read",le="0.2"} 16 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/read",le="0.4"} 16 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/read",le="1"} 16 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/read",le="3"} 16 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/read",le="8"} 16 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/read",le="20"} 16 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/read",le="60"} 16 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/read",le="120"} 16 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/read",le="+Inf"} 16 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/read"} 0.07308876400000001 +prometheus_http_request_duration_seconds_count{handler="/api/v1/read"} 16 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/rules",le="0.1"} 88415 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/rules",le="0.2"} 88627 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/rules",le="0.4"} 88659 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/rules",le="1"} 88661 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/rules",le="3"} 88661 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/rules",le="8"} 88662 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/rules",le="20"} 88662 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/rules",le="60"} 88662 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/rules",le="120"} 88662 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/rules",le="+Inf"} 88662 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/rules"} 629.5237672670012 +prometheus_http_request_duration_seconds_count{handler="/api/v1/rules"} 88662 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/scrape_pools",le="0.1"} 142 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/scrape_pools",le="0.2"} 142 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/scrape_pools",le="0.4"} 142 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/scrape_pools",le="1"} 142 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/scrape_pools",le="3"} 142 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/scrape_pools",le="8"} 142 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/scrape_pools",le="20"} 142 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/scrape_pools",le="60"} 142 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/scrape_pools",le="120"} 142 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/scrape_pools",le="+Inf"} 142 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/scrape_pools"} 0.17501777799999996 +prometheus_http_request_duration_seconds_count{handler="/api/v1/scrape_pools"} 142 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/series",le="0.1"} 1217 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/series",le="0.2"} 1236 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/series",le="0.4"} 1244 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/series",le="1"} 1245 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/series",le="3"} 1245 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/series",le="8"} 1245 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/series",le="20"} 1246 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/series",le="60"} 1246 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/series",le="120"} 1246 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/series",le="+Inf"} 1246 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/series"} 19.03228310300001 +prometheus_http_request_duration_seconds_count{handler="/api/v1/series"} 1246 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/buildinfo",le="0.1"} 4412 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/buildinfo",le="0.2"} 4413 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/buildinfo",le="0.4"} 4413 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/buildinfo",le="1"} 4413 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/buildinfo",le="3"} 4413 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/buildinfo",le="8"} 4413 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/buildinfo",le="20"} 4413 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/buildinfo",le="60"} 4413 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/buildinfo",le="120"} 4413 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/buildinfo",le="+Inf"} 4413 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/status/buildinfo"} 5.222974727000008 +prometheus_http_request_duration_seconds_count{handler="/api/v1/status/buildinfo"} 4413 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/config",le="0.1"} 84669 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/config",le="0.2"} 84716 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/config",le="0.4"} 84721 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/config",le="1"} 84723 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/config",le="3"} 84723 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/config",le="8"} 84723 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/config",le="20"} 84723 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/config",le="60"} 84723 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/config",le="120"} 84723 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/config",le="+Inf"} 84723 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/status/config"} 228.93575751199964 +prometheus_http_request_duration_seconds_count{handler="/api/v1/status/config"} 84723 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/flags",le="0.1"} 32 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/flags",le="0.2"} 32 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/flags",le="0.4"} 32 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/flags",le="1"} 32 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/flags",le="3"} 32 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/flags",le="8"} 32 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/flags",le="20"} 32 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/flags",le="60"} 32 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/flags",le="120"} 32 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/flags",le="+Inf"} 32 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/status/flags"} 0.093302815 +prometheus_http_request_duration_seconds_count{handler="/api/v1/status/flags"} 32 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/runtimeinfo",le="0.1"} 862 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/runtimeinfo",le="0.2"} 863 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/runtimeinfo",le="0.4"} 863 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/runtimeinfo",le="1"} 863 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/runtimeinfo",le="3"} 863 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/runtimeinfo",le="8"} 863 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/runtimeinfo",le="20"} 863 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/runtimeinfo",le="60"} 863 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/runtimeinfo",le="120"} 863 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/runtimeinfo",le="+Inf"} 863 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/status/runtimeinfo"} 8.009312817000001 +prometheus_http_request_duration_seconds_count{handler="/api/v1/status/runtimeinfo"} 863 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/tsdb",le="0.1"} 94 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/tsdb",le="0.2"} 94 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/tsdb",le="0.4"} 94 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/tsdb",le="1"} 94 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/tsdb",le="3"} 94 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/tsdb",le="8"} 94 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/tsdb",le="20"} 94 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/tsdb",le="60"} 94 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/tsdb",le="120"} 94 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/tsdb",le="+Inf"} 94 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/status/tsdb"} 1.3378732679999994 +prometheus_http_request_duration_seconds_count{handler="/api/v1/status/tsdb"} 94 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/walreplay",le="0.1"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/walreplay",le="0.2"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/walreplay",le="0.4"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/walreplay",le="1"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/walreplay",le="3"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/walreplay",le="8"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/walreplay",le="20"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/walreplay",le="60"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/walreplay",le="120"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/walreplay",le="+Inf"} 49 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/status/walreplay"} 0.034643267 +prometheus_http_request_duration_seconds_count{handler="/api/v1/status/walreplay"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/targets",le="0.1"} 191 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/targets",le="0.2"} 191 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/targets",le="0.4"} 191 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/targets",le="1"} 191 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/targets",le="3"} 191 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/targets",le="8"} 191 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/targets",le="20"} 191 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/targets",le="60"} 191 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/targets",le="120"} 191 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/targets",le="+Inf"} 191 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/targets"} 0.5432347889999999 +prometheus_http_request_duration_seconds_count{handler="/api/v1/targets"} 191 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/targets/metadata",le="0.1"} 18 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/targets/metadata",le="0.2"} 18 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/targets/metadata",le="0.4"} 18 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/targets/metadata",le="1"} 18 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/targets/metadata",le="3"} 18 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/targets/metadata",le="8"} 18 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/targets/metadata",le="20"} 18 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/targets/metadata",le="60"} 18 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/targets/metadata",le="120"} 18 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/targets/metadata",le="+Inf"} 18 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/targets/metadata"} 0.043789973 +prometheus_http_request_duration_seconds_count{handler="/api/v1/targets/metadata"} 18 +prometheus_http_request_duration_seconds_bucket{handler="/assets/*filepath",le="0.1"} 385 +prometheus_http_request_duration_seconds_bucket{handler="/assets/*filepath",le="0.2"} 481 +prometheus_http_request_duration_seconds_bucket{handler="/assets/*filepath",le="0.4"} 528 +prometheus_http_request_duration_seconds_bucket{handler="/assets/*filepath",le="1"} 612 +prometheus_http_request_duration_seconds_bucket{handler="/assets/*filepath",le="3"} 634 +prometheus_http_request_duration_seconds_bucket{handler="/assets/*filepath",le="8"} 640 +prometheus_http_request_duration_seconds_bucket{handler="/assets/*filepath",le="20"} 641 +prometheus_http_request_duration_seconds_bucket{handler="/assets/*filepath",le="60"} 642 +prometheus_http_request_duration_seconds_bucket{handler="/assets/*filepath",le="120"} 643 +prometheus_http_request_duration_seconds_bucket{handler="/assets/*filepath",le="+Inf"} 643 +prometheus_http_request_duration_seconds_sum{handler="/assets/*filepath"} 280.0660388799997 +prometheus_http_request_duration_seconds_count{handler="/assets/*filepath"} 643 +prometheus_http_request_duration_seconds_bucket{handler="/classic/static/*filepath",le="0.1"} 103 +prometheus_http_request_duration_seconds_bucket{handler="/classic/static/*filepath",le="0.2"} 103 +prometheus_http_request_duration_seconds_bucket{handler="/classic/static/*filepath",le="0.4"} 103 +prometheus_http_request_duration_seconds_bucket{handler="/classic/static/*filepath",le="1"} 103 +prometheus_http_request_duration_seconds_bucket{handler="/classic/static/*filepath",le="3"} 103 +prometheus_http_request_duration_seconds_bucket{handler="/classic/static/*filepath",le="8"} 103 +prometheus_http_request_duration_seconds_bucket{handler="/classic/static/*filepath",le="20"} 103 +prometheus_http_request_duration_seconds_bucket{handler="/classic/static/*filepath",le="60"} 103 +prometheus_http_request_duration_seconds_bucket{handler="/classic/static/*filepath",le="120"} 103 +prometheus_http_request_duration_seconds_bucket{handler="/classic/static/*filepath",le="+Inf"} 103 +prometheus_http_request_duration_seconds_sum{handler="/classic/static/*filepath"} 0.006113046000000001 +prometheus_http_request_duration_seconds_count{handler="/classic/static/*filepath"} 103 +prometheus_http_request_duration_seconds_bucket{handler="/config",le="0.1"} 13 +prometheus_http_request_duration_seconds_bucket{handler="/config",le="0.2"} 13 +prometheus_http_request_duration_seconds_bucket{handler="/config",le="0.4"} 13 +prometheus_http_request_duration_seconds_bucket{handler="/config",le="1"} 13 +prometheus_http_request_duration_seconds_bucket{handler="/config",le="3"} 13 +prometheus_http_request_duration_seconds_bucket{handler="/config",le="8"} 13 +prometheus_http_request_duration_seconds_bucket{handler="/config",le="20"} 13 +prometheus_http_request_duration_seconds_bucket{handler="/config",le="60"} 13 +prometheus_http_request_duration_seconds_bucket{handler="/config",le="120"} 13 +prometheus_http_request_duration_seconds_bucket{handler="/config",le="+Inf"} 13 +prometheus_http_request_duration_seconds_sum{handler="/config"} 0.0024490289999999997 +prometheus_http_request_duration_seconds_count{handler="/config"} 13 +prometheus_http_request_duration_seconds_bucket{handler="/consoles/*filepath",le="0.1"} 33 +prometheus_http_request_duration_seconds_bucket{handler="/consoles/*filepath",le="0.2"} 34 +prometheus_http_request_duration_seconds_bucket{handler="/consoles/*filepath",le="0.4"} 34 +prometheus_http_request_duration_seconds_bucket{handler="/consoles/*filepath",le="1"} 34 +prometheus_http_request_duration_seconds_bucket{handler="/consoles/*filepath",le="3"} 34 +prometheus_http_request_duration_seconds_bucket{handler="/consoles/*filepath",le="8"} 34 +prometheus_http_request_duration_seconds_bucket{handler="/consoles/*filepath",le="20"} 34 +prometheus_http_request_duration_seconds_bucket{handler="/consoles/*filepath",le="60"} 34 +prometheus_http_request_duration_seconds_bucket{handler="/consoles/*filepath",le="120"} 34 +prometheus_http_request_duration_seconds_bucket{handler="/consoles/*filepath",le="+Inf"} 34 +prometheus_http_request_duration_seconds_sum{handler="/consoles/*filepath"} 0.5689515199999999 +prometheus_http_request_duration_seconds_count{handler="/consoles/*filepath"} 34 +prometheus_http_request_duration_seconds_bucket{handler="/debug/*subpath",le="0.1"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/debug/*subpath",le="0.2"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/debug/*subpath",le="0.4"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/debug/*subpath",le="1"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/debug/*subpath",le="3"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/debug/*subpath",le="8"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/debug/*subpath",le="20"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/debug/*subpath",le="60"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/debug/*subpath",le="120"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/debug/*subpath",le="+Inf"} 4 +prometheus_http_request_duration_seconds_sum{handler="/debug/*subpath"} 0.086499352 +prometheus_http_request_duration_seconds_count{handler="/debug/*subpath"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/favicon.ico",le="0.1"} 177 +prometheus_http_request_duration_seconds_bucket{handler="/favicon.ico",le="0.2"} 177 +prometheus_http_request_duration_seconds_bucket{handler="/favicon.ico",le="0.4"} 177 +prometheus_http_request_duration_seconds_bucket{handler="/favicon.ico",le="1"} 177 +prometheus_http_request_duration_seconds_bucket{handler="/favicon.ico",le="3"} 177 +prometheus_http_request_duration_seconds_bucket{handler="/favicon.ico",le="8"} 177 +prometheus_http_request_duration_seconds_bucket{handler="/favicon.ico",le="20"} 177 +prometheus_http_request_duration_seconds_bucket{handler="/favicon.ico",le="60"} 177 +prometheus_http_request_duration_seconds_bucket{handler="/favicon.ico",le="120"} 177 +prometheus_http_request_duration_seconds_bucket{handler="/favicon.ico",le="+Inf"} 177 +prometheus_http_request_duration_seconds_sum{handler="/favicon.ico"} 0.05591882500000002 +prometheus_http_request_duration_seconds_count{handler="/favicon.ico"} 177 +prometheus_http_request_duration_seconds_bucket{handler="/favicon.svg",le="0.1"} 770 +prometheus_http_request_duration_seconds_bucket{handler="/favicon.svg",le="0.2"} 770 +prometheus_http_request_duration_seconds_bucket{handler="/favicon.svg",le="0.4"} 770 +prometheus_http_request_duration_seconds_bucket{handler="/favicon.svg",le="1"} 770 +prometheus_http_request_duration_seconds_bucket{handler="/favicon.svg",le="3"} 770 +prometheus_http_request_duration_seconds_bucket{handler="/favicon.svg",le="8"} 770 +prometheus_http_request_duration_seconds_bucket{handler="/favicon.svg",le="20"} 770 +prometheus_http_request_duration_seconds_bucket{handler="/favicon.svg",le="60"} 770 +prometheus_http_request_duration_seconds_bucket{handler="/favicon.svg",le="120"} 770 +prometheus_http_request_duration_seconds_bucket{handler="/favicon.svg",le="+Inf"} 770 +prometheus_http_request_duration_seconds_sum{handler="/favicon.svg"} 0.3058455699999999 +prometheus_http_request_duration_seconds_count{handler="/favicon.svg"} 770 +prometheus_http_request_duration_seconds_bucket{handler="/federate",le="0.1"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/federate",le="0.2"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/federate",le="0.4"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/federate",le="1"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/federate",le="3"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/federate",le="8"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/federate",le="20"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/federate",le="60"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/federate",le="120"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/federate",le="+Inf"} 4 +prometheus_http_request_duration_seconds_sum{handler="/federate"} 0.05996980699999999 +prometheus_http_request_duration_seconds_count{handler="/federate"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/flags",le="0.1"} 9 +prometheus_http_request_duration_seconds_bucket{handler="/flags",le="0.2"} 9 +prometheus_http_request_duration_seconds_bucket{handler="/flags",le="0.4"} 9 +prometheus_http_request_duration_seconds_bucket{handler="/flags",le="1"} 9 +prometheus_http_request_duration_seconds_bucket{handler="/flags",le="3"} 9 +prometheus_http_request_duration_seconds_bucket{handler="/flags",le="8"} 9 +prometheus_http_request_duration_seconds_bucket{handler="/flags",le="20"} 9 +prometheus_http_request_duration_seconds_bucket{handler="/flags",le="60"} 9 +prometheus_http_request_duration_seconds_bucket{handler="/flags",le="120"} 9 +prometheus_http_request_duration_seconds_bucket{handler="/flags",le="+Inf"} 9 +prometheus_http_request_duration_seconds_sum{handler="/flags"} 0.001586781 +prometheus_http_request_duration_seconds_count{handler="/flags"} 9 +prometheus_http_request_duration_seconds_bucket{handler="/graph",le="0.1"} 751 +prometheus_http_request_duration_seconds_bucket{handler="/graph",le="0.2"} 751 +prometheus_http_request_duration_seconds_bucket{handler="/graph",le="0.4"} 751 +prometheus_http_request_duration_seconds_bucket{handler="/graph",le="1"} 751 +prometheus_http_request_duration_seconds_bucket{handler="/graph",le="3"} 751 +prometheus_http_request_duration_seconds_bucket{handler="/graph",le="8"} 751 +prometheus_http_request_duration_seconds_bucket{handler="/graph",le="20"} 751 +prometheus_http_request_duration_seconds_bucket{handler="/graph",le="60"} 751 +prometheus_http_request_duration_seconds_bucket{handler="/graph",le="120"} 751 +prometheus_http_request_duration_seconds_bucket{handler="/graph",le="+Inf"} 751 +prometheus_http_request_duration_seconds_sum{handler="/graph"} 0.032583203000000005 +prometheus_http_request_duration_seconds_count{handler="/graph"} 751 +prometheus_http_request_duration_seconds_bucket{handler="/metrics",le="0.1"} 3.988654e+06 +prometheus_http_request_duration_seconds_bucket{handler="/metrics",le="0.2"} 4.052461e+06 +prometheus_http_request_duration_seconds_bucket{handler="/metrics",le="0.4"} 4.057771e+06 +prometheus_http_request_duration_seconds_bucket{handler="/metrics",le="1"} 4.058809e+06 +prometheus_http_request_duration_seconds_bucket{handler="/metrics",le="3"} 4.059071e+06 +prometheus_http_request_duration_seconds_bucket{handler="/metrics",le="8"} 4.059092e+06 +prometheus_http_request_duration_seconds_bucket{handler="/metrics",le="20"} 4.059092e+06 +prometheus_http_request_duration_seconds_bucket{handler="/metrics",le="60"} 4.059092e+06 +prometheus_http_request_duration_seconds_bucket{handler="/metrics",le="120"} 4.059092e+06 +prometheus_http_request_duration_seconds_bucket{handler="/metrics",le="+Inf"} 4.059092e+06 +prometheus_http_request_duration_seconds_sum{handler="/metrics"} 107584.12561354278 +prometheus_http_request_duration_seconds_count{handler="/metrics"} 4.059092e+06 +prometheus_http_request_duration_seconds_bucket{handler="/query",le="0.1"} 1774 +prometheus_http_request_duration_seconds_bucket{handler="/query",le="0.2"} 1774 +prometheus_http_request_duration_seconds_bucket{handler="/query",le="0.4"} 1774 +prometheus_http_request_duration_seconds_bucket{handler="/query",le="1"} 1774 +prometheus_http_request_duration_seconds_bucket{handler="/query",le="3"} 1774 +prometheus_http_request_duration_seconds_bucket{handler="/query",le="8"} 1774 +prometheus_http_request_duration_seconds_bucket{handler="/query",le="20"} 1774 +prometheus_http_request_duration_seconds_bucket{handler="/query",le="60"} 1774 +prometheus_http_request_duration_seconds_bucket{handler="/query",le="120"} 1774 +prometheus_http_request_duration_seconds_bucket{handler="/query",le="+Inf"} 1774 +prometheus_http_request_duration_seconds_sum{handler="/query"} 0.5220201759999993 +prometheus_http_request_duration_seconds_count{handler="/query"} 1774 +prometheus_http_request_duration_seconds_bucket{handler="/rules",le="0.1"} 8672 +prometheus_http_request_duration_seconds_bucket{handler="/rules",le="0.2"} 8672 +prometheus_http_request_duration_seconds_bucket{handler="/rules",le="0.4"} 8673 +prometheus_http_request_duration_seconds_bucket{handler="/rules",le="1"} 8673 +prometheus_http_request_duration_seconds_bucket{handler="/rules",le="3"} 8673 +prometheus_http_request_duration_seconds_bucket{handler="/rules",le="8"} 8673 +prometheus_http_request_duration_seconds_bucket{handler="/rules",le="20"} 8673 +prometheus_http_request_duration_seconds_bucket{handler="/rules",le="60"} 8673 +prometheus_http_request_duration_seconds_bucket{handler="/rules",le="120"} 8673 +prometheus_http_request_duration_seconds_bucket{handler="/rules",le="+Inf"} 8673 +prometheus_http_request_duration_seconds_sum{handler="/rules"} 2.776021043000005 +prometheus_http_request_duration_seconds_count{handler="/rules"} 8673 +prometheus_http_request_duration_seconds_bucket{handler="/service-discovery",le="0.1"} 20 +prometheus_http_request_duration_seconds_bucket{handler="/service-discovery",le="0.2"} 20 +prometheus_http_request_duration_seconds_bucket{handler="/service-discovery",le="0.4"} 20 +prometheus_http_request_duration_seconds_bucket{handler="/service-discovery",le="1"} 20 +prometheus_http_request_duration_seconds_bucket{handler="/service-discovery",le="3"} 20 +prometheus_http_request_duration_seconds_bucket{handler="/service-discovery",le="8"} 20 +prometheus_http_request_duration_seconds_bucket{handler="/service-discovery",le="20"} 20 +prometheus_http_request_duration_seconds_bucket{handler="/service-discovery",le="60"} 20 +prometheus_http_request_duration_seconds_bucket{handler="/service-discovery",le="120"} 20 +prometheus_http_request_duration_seconds_bucket{handler="/service-discovery",le="+Inf"} 20 +prometheus_http_request_duration_seconds_sum{handler="/service-discovery"} 0.004057062 +prometheus_http_request_duration_seconds_count{handler="/service-discovery"} 20 +prometheus_http_request_duration_seconds_bucket{handler="/status",le="0.1"} 46 +prometheus_http_request_duration_seconds_bucket{handler="/status",le="0.2"} 46 +prometheus_http_request_duration_seconds_bucket{handler="/status",le="0.4"} 46 +prometheus_http_request_duration_seconds_bucket{handler="/status",le="1"} 46 +prometheus_http_request_duration_seconds_bucket{handler="/status",le="3"} 46 +prometheus_http_request_duration_seconds_bucket{handler="/status",le="8"} 46 +prometheus_http_request_duration_seconds_bucket{handler="/status",le="20"} 46 +prometheus_http_request_duration_seconds_bucket{handler="/status",le="60"} 46 +prometheus_http_request_duration_seconds_bucket{handler="/status",le="120"} 46 +prometheus_http_request_duration_seconds_bucket{handler="/status",le="+Inf"} 46 +prometheus_http_request_duration_seconds_sum{handler="/status"} 0.010107473 +prometheus_http_request_duration_seconds_count{handler="/status"} 46 +prometheus_http_request_duration_seconds_bucket{handler="/targets",le="0.1"} 39 +prometheus_http_request_duration_seconds_bucket{handler="/targets",le="0.2"} 39 +prometheus_http_request_duration_seconds_bucket{handler="/targets",le="0.4"} 39 +prometheus_http_request_duration_seconds_bucket{handler="/targets",le="1"} 39 +prometheus_http_request_duration_seconds_bucket{handler="/targets",le="3"} 39 +prometheus_http_request_duration_seconds_bucket{handler="/targets",le="8"} 39 +prometheus_http_request_duration_seconds_bucket{handler="/targets",le="20"} 39 +prometheus_http_request_duration_seconds_bucket{handler="/targets",le="60"} 39 +prometheus_http_request_duration_seconds_bucket{handler="/targets",le="120"} 39 +prometheus_http_request_duration_seconds_bucket{handler="/targets",le="+Inf"} 39 +prometheus_http_request_duration_seconds_sum{handler="/targets"} 0.009001180000000001 +prometheus_http_request_duration_seconds_count{handler="/targets"} 39 +prometheus_http_request_duration_seconds_bucket{handler="/tsdb-status",le="0.1"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/tsdb-status",le="0.2"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/tsdb-status",le="0.4"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/tsdb-status",le="1"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/tsdb-status",le="3"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/tsdb-status",le="8"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/tsdb-status",le="20"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/tsdb-status",le="60"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/tsdb-status",le="120"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/tsdb-status",le="+Inf"} 49 +prometheus_http_request_duration_seconds_sum{handler="/tsdb-status"} 0.018204165 +prometheus_http_request_duration_seconds_count{handler="/tsdb-status"} 49 +prometheus_http_requests_total{code="200",handler="/"} 0 +prometheus_http_requests_total{code="200",handler="/-/healthy"} 29524 +prometheus_http_requests_total{code="200",handler="/-/quit"} 0 +prometheus_http_requests_total{code="200",handler="/-/ready"} 49 +prometheus_http_requests_total{code="200",handler="/-/reload"} 0 +prometheus_http_requests_total{code="200",handler="/alertmanager-discovery"} 0 +prometheus_http_requests_total{code="200",handler="/alerts"} 48 +prometheus_http_requests_total{code="200",handler="/api/v1/*path"} 0 +prometheus_http_requests_total{code="200",handler="/api/v1/admin/tsdb/clean_tombstones"} 0 +prometheus_http_requests_total{code="200",handler="/api/v1/admin/tsdb/delete_series"} 0 +prometheus_http_requests_total{code="200",handler="/api/v1/admin/tsdb/snapshot"} 0 +prometheus_http_requests_total{code="200",handler="/api/v1/alertmanagers"} 8 +prometheus_http_requests_total{code="200",handler="/api/v1/alerts"} 14635 +prometheus_http_requests_total{code="200",handler="/api/v1/format_query"} 3 +prometheus_http_requests_total{code="200",handler="/api/v1/label/:name/values"} 14679 +prometheus_http_requests_total{code="200",handler="/api/v1/labels"} 4915 +prometheus_http_requests_total{code="200",handler="/api/v1/metadata"} 2093 +prometheus_http_requests_total{code="200",handler="/api/v1/notifications"} 12 +prometheus_http_requests_total{code="200",handler="/api/v1/notifications/live"} 4529 +prometheus_http_requests_total{code="200",handler="/api/v1/otlp/v1/metrics"} 0 +prometheus_http_requests_total{code="200",handler="/api/v1/parse_query"} 247 +prometheus_http_requests_total{code="200",handler="/api/v1/query"} 326411 +prometheus_http_requests_total{code="200",handler="/api/v1/query_exemplars"} 841 +prometheus_http_requests_total{code="200",handler="/api/v1/query_range"} 1.183734e+06 +prometheus_http_requests_total{code="200",handler="/api/v1/read"} 16 +prometheus_http_requests_total{code="200",handler="/api/v1/rules"} 88658 +prometheus_http_requests_total{code="200",handler="/api/v1/scrape_pools"} 142 +prometheus_http_requests_total{code="200",handler="/api/v1/series"} 1235 +prometheus_http_requests_total{code="200",handler="/api/v1/status/buildinfo"} 4413 +prometheus_http_requests_total{code="200",handler="/api/v1/status/config"} 84722 +prometheus_http_requests_total{code="200",handler="/api/v1/status/flags"} 32 +prometheus_http_requests_total{code="200",handler="/api/v1/status/runtimeinfo"} 863 +prometheus_http_requests_total{code="200",handler="/api/v1/status/tsdb"} 94 +prometheus_http_requests_total{code="200",handler="/api/v1/status/walreplay"} 49 +prometheus_http_requests_total{code="200",handler="/api/v1/targets"} 191 +prometheus_http_requests_total{code="200",handler="/api/v1/targets/metadata"} 16 +prometheus_http_requests_total{code="200",handler="/api/v1/write"} 0 +prometheus_http_requests_total{code="200",handler="/assets/*filepath"} 2213 +prometheus_http_requests_total{code="200",handler="/classic/static/*filepath"} 0 +prometheus_http_requests_total{code="200",handler="/config"} 13 +prometheus_http_requests_total{code="200",handler="/consoles/*filepath"} 17 +prometheus_http_requests_total{code="200",handler="/debug/*subpath"} 3 +prometheus_http_requests_total{code="200",handler="/favicon.ico"} 0 +prometheus_http_requests_total{code="200",handler="/favicon.svg"} 769 +prometheus_http_requests_total{code="200",handler="/federate"} 4 +prometheus_http_requests_total{code="200",handler="/flags"} 9 +prometheus_http_requests_total{code="200",handler="/graph"} 0 +prometheus_http_requests_total{code="200",handler="/manifest.json"} 0 +prometheus_http_requests_total{code="200",handler="/metrics"} 4.059092e+06 +prometheus_http_requests_total{code="200",handler="/query"} 1774 +prometheus_http_requests_total{code="200",handler="/rules"} 8673 +prometheus_http_requests_total{code="200",handler="/service-discovery"} 20 +prometheus_http_requests_total{code="200",handler="/status"} 46 +prometheus_http_requests_total{code="200",handler="/targets"} 39 +prometheus_http_requests_total{code="200",handler="/tsdb-status"} 49 +prometheus_http_requests_total{code="200",handler="/version"} 0 +prometheus_http_requests_total{code="204",handler="/api/v1/*path"} 27 +prometheus_http_requests_total{code="204",handler="/api/v1/notifications/live"} 5 +prometheus_http_requests_total{code="301",handler="/debug/*subpath"} 1 +prometheus_http_requests_total{code="302",handler="/"} 688 +prometheus_http_requests_total{code="302",handler="/graph"} 751 +prometheus_http_requests_total{code="400",handler="/api/v1/format_query"} 1 +prometheus_http_requests_total{code="400",handler="/api/v1/label/:name/values"} 3292 +prometheus_http_requests_total{code="400",handler="/api/v1/labels"} 6 +prometheus_http_requests_total{code="400",handler="/api/v1/parse_query"} 18 +prometheus_http_requests_total{code="400",handler="/api/v1/query"} 155 +prometheus_http_requests_total{code="400",handler="/api/v1/query_range"} 263 +prometheus_http_requests_total{code="400",handler="/api/v1/rules"} 4 +prometheus_http_requests_total{code="400",handler="/api/v1/series"} 11 +prometheus_http_requests_total{code="400",handler="/api/v1/targets/metadata"} 2 +prometheus_http_requests_total{code="404",handler="/assets/*filepath"} 12 +prometheus_http_requests_total{code="404",handler="/classic/static/*filepath"} 103 +prometheus_http_requests_total{code="404",handler="/consoles/*filepath"} 17 +prometheus_http_requests_total{code="404",handler="/favicon.ico"} 177 +prometheus_http_requests_total{code="416",handler="/favicon.svg"} 1 +prometheus_http_requests_total{code="422",handler="/api/v1/query"} 114 +prometheus_http_requests_total{code="422",handler="/api/v1/query_range"} 31 +prometheus_http_requests_total{code="499",handler="/api/v1/query"} 7 +prometheus_http_requests_total{code="499",handler="/api/v1/query_range"} 20 +prometheus_http_requests_total{code="503",handler="/api/v1/query"} 7 +prometheus_http_requests_total{code="503",handler="/api/v1/query_range"} 4 +prometheus_http_requests_total{code="503",handler="/api/v1/status/config"} 1 +prometheus_http_response_size_bytes_bucket{handler="/",le="100"} 688 +prometheus_http_response_size_bytes_bucket{handler="/",le="1000"} 688 +prometheus_http_response_size_bytes_bucket{handler="/",le="10000"} 688 +prometheus_http_response_size_bytes_bucket{handler="/",le="100000"} 688 +prometheus_http_response_size_bytes_bucket{handler="/",le="1e+06"} 688 +prometheus_http_response_size_bytes_bucket{handler="/",le="1e+07"} 688 +prometheus_http_response_size_bytes_bucket{handler="/",le="1e+08"} 688 +prometheus_http_response_size_bytes_bucket{handler="/",le="1e+09"} 688 +prometheus_http_response_size_bytes_bucket{handler="/",le="+Inf"} 688 +prometheus_http_response_size_bytes_sum{handler="/"} 19952 +prometheus_http_response_size_bytes_count{handler="/"} 688 +prometheus_http_response_size_bytes_bucket{handler="/-/healthy",le="100"} 29524 +prometheus_http_response_size_bytes_bucket{handler="/-/healthy",le="1000"} 29524 +prometheus_http_response_size_bytes_bucket{handler="/-/healthy",le="10000"} 29524 +prometheus_http_response_size_bytes_bucket{handler="/-/healthy",le="100000"} 29524 +prometheus_http_response_size_bytes_bucket{handler="/-/healthy",le="1e+06"} 29524 +prometheus_http_response_size_bytes_bucket{handler="/-/healthy",le="1e+07"} 29524 +prometheus_http_response_size_bytes_bucket{handler="/-/healthy",le="1e+08"} 29524 +prometheus_http_response_size_bytes_bucket{handler="/-/healthy",le="1e+09"} 29524 +prometheus_http_response_size_bytes_bucket{handler="/-/healthy",le="+Inf"} 29524 +prometheus_http_response_size_bytes_sum{handler="/-/healthy"} 885720 +prometheus_http_response_size_bytes_count{handler="/-/healthy"} 29524 +prometheus_http_response_size_bytes_bucket{handler="/-/ready",le="100"} 49 +prometheus_http_response_size_bytes_bucket{handler="/-/ready",le="1000"} 49 +prometheus_http_response_size_bytes_bucket{handler="/-/ready",le="10000"} 49 +prometheus_http_response_size_bytes_bucket{handler="/-/ready",le="100000"} 49 +prometheus_http_response_size_bytes_bucket{handler="/-/ready",le="1e+06"} 49 +prometheus_http_response_size_bytes_bucket{handler="/-/ready",le="1e+07"} 49 +prometheus_http_response_size_bytes_bucket{handler="/-/ready",le="1e+08"} 49 +prometheus_http_response_size_bytes_bucket{handler="/-/ready",le="1e+09"} 49 +prometheus_http_response_size_bytes_bucket{handler="/-/ready",le="+Inf"} 49 +prometheus_http_response_size_bytes_sum{handler="/-/ready"} 1372 +prometheus_http_response_size_bytes_count{handler="/-/ready"} 49 +prometheus_http_response_size_bytes_bucket{handler="/alerts",le="100"} 0 +prometheus_http_response_size_bytes_bucket{handler="/alerts",le="1000"} 0 +prometheus_http_response_size_bytes_bucket{handler="/alerts",le="10000"} 48 +prometheus_http_response_size_bytes_bucket{handler="/alerts",le="100000"} 48 +prometheus_http_response_size_bytes_bucket{handler="/alerts",le="1e+06"} 48 +prometheus_http_response_size_bytes_bucket{handler="/alerts",le="1e+07"} 48 +prometheus_http_response_size_bytes_bucket{handler="/alerts",le="1e+08"} 48 +prometheus_http_response_size_bytes_bucket{handler="/alerts",le="1e+09"} 48 +prometheus_http_response_size_bytes_bucket{handler="/alerts",le="+Inf"} 48 +prometheus_http_response_size_bytes_sum{handler="/alerts"} 84096 +prometheus_http_response_size_bytes_count{handler="/alerts"} 48 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/*path",le="100"} 27 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/*path",le="1000"} 27 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/*path",le="10000"} 27 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/*path",le="100000"} 27 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/*path",le="1e+06"} 27 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/*path",le="1e+07"} 27 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/*path",le="1e+08"} 27 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/*path",le="1e+09"} 27 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/*path",le="+Inf"} 27 +prometheus_http_response_size_bytes_sum{handler="/api/v1/*path"} 0 +prometheus_http_response_size_bytes_count{handler="/api/v1/*path"} 27 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/alertmanagers",le="100"} 0 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/alertmanagers",le="1000"} 8 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/alertmanagers",le="10000"} 8 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/alertmanagers",le="100000"} 8 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/alertmanagers",le="1e+06"} 8 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/alertmanagers",le="1e+07"} 8 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/alertmanagers",le="1e+08"} 8 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/alertmanagers",le="1e+09"} 8 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/alertmanagers",le="+Inf"} 8 +prometheus_http_response_size_bytes_sum{handler="/api/v1/alertmanagers"} 1064 +prometheus_http_response_size_bytes_count{handler="/api/v1/alertmanagers"} 8 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/alerts",le="100"} 0 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/alerts",le="1000"} 14635 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/alerts",le="10000"} 14635 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/alerts",le="100000"} 14635 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/alerts",le="1e+06"} 14635 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/alerts",le="1e+07"} 14635 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/alerts",le="1e+08"} 14635 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/alerts",le="1e+09"} 14635 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/alerts",le="+Inf"} 14635 +prometheus_http_response_size_bytes_sum{handler="/api/v1/alerts"} 1.0260926e+07 +prometheus_http_response_size_bytes_count{handler="/api/v1/alerts"} 14635 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/format_query",le="100"} 0 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/format_query",le="1000"} 4 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/format_query",le="10000"} 4 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/format_query",le="100000"} 4 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/format_query",le="1e+06"} 4 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/format_query",le="1e+07"} 4 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/format_query",le="1e+08"} 4 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/format_query",le="1e+09"} 4 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/format_query",le="+Inf"} 4 +prometheus_http_response_size_bytes_sum{handler="/api/v1/format_query"} 495 +prometheus_http_response_size_bytes_count{handler="/api/v1/format_query"} 4 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/label/:name/values",le="100"} 8167 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/label/:name/values",le="1000"} 15939 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/label/:name/values",le="10000"} 17901 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/label/:name/values",le="100000"} 17971 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/label/:name/values",le="1e+06"} 17971 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/label/:name/values",le="1e+07"} 17971 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/label/:name/values",le="1e+08"} 17971 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/label/:name/values",le="1e+09"} 17971 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/label/:name/values",le="+Inf"} 17971 +prometheus_http_response_size_bytes_sum{handler="/api/v1/label/:name/values"} 2.0009454e+07 +prometheus_http_response_size_bytes_count{handler="/api/v1/label/:name/values"} 17971 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/labels",le="100"} 102 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/labels",le="1000"} 4908 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/labels",le="10000"} 4921 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/labels",le="100000"} 4921 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/labels",le="1e+06"} 4921 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/labels",le="1e+07"} 4921 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/labels",le="1e+08"} 4921 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/labels",le="1e+09"} 4921 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/labels",le="+Inf"} 4921 +prometheus_http_response_size_bytes_sum{handler="/api/v1/labels"} 3.16046e+06 +prometheus_http_response_size_bytes_count{handler="/api/v1/labels"} 4921 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/metadata",le="100"} 170 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/metadata",le="1000"} 1368 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/metadata",le="10000"} 1368 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/metadata",le="100000"} 2093 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/metadata",le="1e+06"} 2093 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/metadata",le="1e+07"} 2093 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/metadata",le="1e+08"} 2093 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/metadata",le="1e+09"} 2093 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/metadata",le="+Inf"} 2093 +prometheus_http_response_size_bytes_sum{handler="/api/v1/metadata"} 1.5950362e+07 +prometheus_http_response_size_bytes_count{handler="/api/v1/metadata"} 2093 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/notifications",le="100"} 12 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/notifications",le="1000"} 12 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/notifications",le="10000"} 12 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/notifications",le="100000"} 12 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/notifications",le="1e+06"} 12 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/notifications",le="1e+07"} 12 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/notifications",le="1e+08"} 12 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/notifications",le="1e+09"} 12 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/notifications",le="+Inf"} 12 +prometheus_http_response_size_bytes_sum{handler="/api/v1/notifications"} 360 +prometheus_http_response_size_bytes_count{handler="/api/v1/notifications"} 12 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/notifications/live",le="100"} 4529 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/notifications/live",le="1000"} 4534 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/notifications/live",le="10000"} 4534 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/notifications/live",le="100000"} 4534 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/notifications/live",le="1e+06"} 4534 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/notifications/live",le="1e+07"} 4534 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/notifications/live",le="1e+08"} 4534 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/notifications/live",le="1e+09"} 4534 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/notifications/live",le="+Inf"} 4534 +prometheus_http_response_size_bytes_sum{handler="/api/v1/notifications/live"} 1365 +prometheus_http_response_size_bytes_count{handler="/api/v1/notifications/live"} 4534 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/parse_query",le="100"} 0 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/parse_query",le="1000"} 265 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/parse_query",le="10000"} 265 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/parse_query",le="100000"} 265 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/parse_query",le="1e+06"} 265 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/parse_query",le="1e+07"} 265 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/parse_query",le="1e+08"} 265 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/parse_query",le="1e+09"} 265 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/parse_query",le="+Inf"} 265 +prometheus_http_response_size_bytes_sum{handler="/api/v1/parse_query"} 67620 +prometheus_http_response_size_bytes_count{handler="/api/v1/parse_query"} 265 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query",le="100"} 25147 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query",le="1000"} 325005 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query",le="10000"} 325143 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query",le="100000"} 326635 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query",le="1e+06"} 326692 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query",le="1e+07"} 326694 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query",le="1e+08"} 326694 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query",le="1e+09"} 326694 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query",le="+Inf"} 326694 +prometheus_http_response_size_bytes_sum{handler="/api/v1/query"} 2.34481756e+08 +prometheus_http_response_size_bytes_count{handler="/api/v1/query"} 326694 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query_exemplars",le="100"} 841 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query_exemplars",le="1000"} 841 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query_exemplars",le="10000"} 841 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query_exemplars",le="100000"} 841 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query_exemplars",le="1e+06"} 841 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query_exemplars",le="1e+07"} 841 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query_exemplars",le="1e+08"} 841 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query_exemplars",le="1e+09"} 841 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query_exemplars",le="+Inf"} 841 +prometheus_http_response_size_bytes_sum{handler="/api/v1/query_exemplars"} 50460 +prometheus_http_response_size_bytes_count{handler="/api/v1/query_exemplars"} 841 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query_range",le="100"} 69552 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query_range",le="1000"} 917165 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query_range",le="10000"} 1.152103e+06 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query_range",le="100000"} 1.183391e+06 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query_range",le="1e+06"} 1.183621e+06 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query_range",le="1e+07"} 1.184052e+06 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query_range",le="1e+08"} 1.184052e+06 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query_range",le="1e+09"} 1.184052e+06 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query_range",le="+Inf"} 1.184052e+06 +prometheus_http_response_size_bytes_sum{handler="/api/v1/query_range"} 1.869092376e+09 +prometheus_http_response_size_bytes_count{handler="/api/v1/query_range"} 1.184052e+06 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/read",le="100"} 0 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/read",le="1000"} 0 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/read",le="10000"} 16 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/read",le="100000"} 16 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/read",le="1e+06"} 16 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/read",le="1e+07"} 16 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/read",le="1e+08"} 16 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/read",le="1e+09"} 16 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/read",le="+Inf"} 16 +prometheus_http_response_size_bytes_sum{handler="/api/v1/read"} 50800 +prometheus_http_response_size_bytes_count{handler="/api/v1/read"} 16 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/rules",le="100"} 0 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/rules",le="1000"} 4 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/rules",le="10000"} 88656 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/rules",le="100000"} 88662 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/rules",le="1e+06"} 88662 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/rules",le="1e+07"} 88662 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/rules",le="1e+08"} 88662 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/rules",le="1e+09"} 88662 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/rules",le="+Inf"} 88662 +prometheus_http_response_size_bytes_sum{handler="/api/v1/rules"} 7.01925022e+08 +prometheus_http_response_size_bytes_count{handler="/api/v1/rules"} 88662 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/scrape_pools",le="100"} 0 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/scrape_pools",le="1000"} 142 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/scrape_pools",le="10000"} 142 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/scrape_pools",le="100000"} 142 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/scrape_pools",le="1e+06"} 142 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/scrape_pools",le="1e+07"} 142 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/scrape_pools",le="1e+08"} 142 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/scrape_pools",le="1e+09"} 142 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/scrape_pools",le="+Inf"} 142 +prometheus_http_response_size_bytes_sum{handler="/api/v1/scrape_pools"} 18318 +prometheus_http_response_size_bytes_count{handler="/api/v1/scrape_pools"} 142 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/series",le="100"} 1011 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/series",le="1000"} 1233 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/series",le="10000"} 1244 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/series",le="100000"} 1246 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/series",le="1e+06"} 1246 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/series",le="1e+07"} 1246 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/series",le="1e+08"} 1246 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/series",le="1e+09"} 1246 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/series",le="+Inf"} 1246 +prometheus_http_response_size_bytes_sum{handler="/api/v1/series"} 235829 +prometheus_http_response_size_bytes_count{handler="/api/v1/series"} 1246 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/buildinfo",le="100"} 0 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/buildinfo",le="1000"} 4413 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/buildinfo",le="10000"} 4413 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/buildinfo",le="100000"} 4413 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/buildinfo",le="1e+06"} 4413 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/buildinfo",le="1e+07"} 4413 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/buildinfo",le="1e+08"} 4413 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/buildinfo",le="1e+09"} 4413 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/buildinfo",le="+Inf"} 4413 +prometheus_http_response_size_bytes_sum{handler="/api/v1/status/buildinfo"} 807579 +prometheus_http_response_size_bytes_count{handler="/api/v1/status/buildinfo"} 4413 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/config",le="100"} 1 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/config",le="1000"} 84723 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/config",le="10000"} 84723 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/config",le="100000"} 84723 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/config",le="1e+06"} 84723 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/config",le="1e+07"} 84723 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/config",le="1e+08"} 84723 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/config",le="1e+09"} 84723 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/config",le="+Inf"} 84723 +prometheus_http_response_size_bytes_sum{handler="/api/v1/status/config"} 6.3710963e+07 +prometheus_http_response_size_bytes_count{handler="/api/v1/status/config"} 84723 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/flags",le="100"} 0 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/flags",le="1000"} 32 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/flags",le="10000"} 32 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/flags",le="100000"} 32 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/flags",le="1e+06"} 32 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/flags",le="1e+07"} 32 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/flags",le="1e+08"} 32 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/flags",le="1e+09"} 32 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/flags",le="+Inf"} 32 +prometheus_http_response_size_bytes_sum{handler="/api/v1/status/flags"} 31968 +prometheus_http_response_size_bytes_count{handler="/api/v1/status/flags"} 32 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/runtimeinfo",le="100"} 0 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/runtimeinfo",le="1000"} 863 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/runtimeinfo",le="10000"} 863 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/runtimeinfo",le="100000"} 863 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/runtimeinfo",le="1e+06"} 863 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/runtimeinfo",le="1e+07"} 863 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/runtimeinfo",le="1e+08"} 863 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/runtimeinfo",le="1e+09"} 863 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/runtimeinfo",le="+Inf"} 863 +prometheus_http_response_size_bytes_sum{handler="/api/v1/status/runtimeinfo"} 240400 +prometheus_http_response_size_bytes_count{handler="/api/v1/status/runtimeinfo"} 863 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/tsdb",le="100"} 0 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/tsdb",le="1000"} 92 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/tsdb",le="10000"} 94 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/tsdb",le="100000"} 94 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/tsdb",le="1e+06"} 94 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/tsdb",le="1e+07"} 94 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/tsdb",le="1e+08"} 94 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/tsdb",le="1e+09"} 94 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/tsdb",le="+Inf"} 94 +prometheus_http_response_size_bytes_sum{handler="/api/v1/status/tsdb"} 66441 +prometheus_http_response_size_bytes_count{handler="/api/v1/status/tsdb"} 94 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/walreplay",le="100"} 49 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/walreplay",le="1000"} 49 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/walreplay",le="10000"} 49 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/walreplay",le="100000"} 49 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/walreplay",le="1e+06"} 49 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/walreplay",le="1e+07"} 49 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/walreplay",le="1e+08"} 49 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/walreplay",le="1e+09"} 49 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/walreplay",le="+Inf"} 49 +prometheus_http_response_size_bytes_sum{handler="/api/v1/status/walreplay"} 3381 +prometheus_http_response_size_bytes_count{handler="/api/v1/status/walreplay"} 49 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/targets",le="100"} 0 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/targets",le="1000"} 184 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/targets",le="10000"} 191 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/targets",le="100000"} 191 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/targets",le="1e+06"} 191 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/targets",le="1e+07"} 191 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/targets",le="1e+08"} 191 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/targets",le="1e+09"} 191 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/targets",le="+Inf"} 191 +prometheus_http_response_size_bytes_sum{handler="/api/v1/targets"} 191150 +prometheus_http_response_size_bytes_count{handler="/api/v1/targets"} 191 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/targets/metadata",le="100"} 1 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/targets/metadata",le="1000"} 18 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/targets/metadata",le="10000"} 18 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/targets/metadata",le="100000"} 18 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/targets/metadata",le="1e+06"} 18 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/targets/metadata",le="1e+07"} 18 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/targets/metadata",le="1e+08"} 18 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/targets/metadata",le="1e+09"} 18 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/targets/metadata",le="+Inf"} 18 +prometheus_http_response_size_bytes_sum{handler="/api/v1/targets/metadata"} 2736 +prometheus_http_response_size_bytes_count{handler="/api/v1/targets/metadata"} 18 +prometheus_http_response_size_bytes_bucket{handler="/assets/*filepath",le="100"} 12 +prometheus_http_response_size_bytes_bucket{handler="/assets/*filepath",le="1000"} 12 +prometheus_http_response_size_bytes_bucket{handler="/assets/*filepath",le="10000"} 13 +prometheus_http_response_size_bytes_bucket{handler="/assets/*filepath",le="100000"} 142 +prometheus_http_response_size_bytes_bucket{handler="/assets/*filepath",le="1e+06"} 958 +prometheus_http_response_size_bytes_bucket{handler="/assets/*filepath",le="1e+07"} 2225 +prometheus_http_response_size_bytes_bucket{handler="/assets/*filepath",le="1e+08"} 2225 +prometheus_http_response_size_bytes_bucket{handler="/assets/*filepath",le="1e+09"} 2225 +prometheus_http_response_size_bytes_bucket{handler="/assets/*filepath",le="+Inf"} 2225 +prometheus_http_response_size_bytes_sum{handler="/assets/*filepath"} 3.525958804e+09 +prometheus_http_response_size_bytes_count{handler="/assets/*filepath"} 2225 +prometheus_http_response_size_bytes_bucket{handler="/classic/static/*filepath",le="100"} 103 +prometheus_http_response_size_bytes_bucket{handler="/classic/static/*filepath",le="1000"} 103 +prometheus_http_response_size_bytes_bucket{handler="/classic/static/*filepath",le="10000"} 103 +prometheus_http_response_size_bytes_bucket{handler="/classic/static/*filepath",le="100000"} 103 +prometheus_http_response_size_bytes_bucket{handler="/classic/static/*filepath",le="1e+06"} 103 +prometheus_http_response_size_bytes_bucket{handler="/classic/static/*filepath",le="1e+07"} 103 +prometheus_http_response_size_bytes_bucket{handler="/classic/static/*filepath",le="1e+08"} 103 +prometheus_http_response_size_bytes_bucket{handler="/classic/static/*filepath",le="1e+09"} 103 +prometheus_http_response_size_bytes_bucket{handler="/classic/static/*filepath",le="+Inf"} 103 +prometheus_http_response_size_bytes_sum{handler="/classic/static/*filepath"} 1957 +prometheus_http_response_size_bytes_count{handler="/classic/static/*filepath"} 103 +prometheus_http_response_size_bytes_bucket{handler="/config",le="100"} 0 +prometheus_http_response_size_bytes_bucket{handler="/config",le="1000"} 0 +prometheus_http_response_size_bytes_bucket{handler="/config",le="10000"} 13 +prometheus_http_response_size_bytes_bucket{handler="/config",le="100000"} 13 +prometheus_http_response_size_bytes_bucket{handler="/config",le="1e+06"} 13 +prometheus_http_response_size_bytes_bucket{handler="/config",le="1e+07"} 13 +prometheus_http_response_size_bytes_bucket{handler="/config",le="1e+08"} 13 +prometheus_http_response_size_bytes_bucket{handler="/config",le="1e+09"} 13 +prometheus_http_response_size_bytes_bucket{handler="/config",le="+Inf"} 13 +prometheus_http_response_size_bytes_sum{handler="/config"} 22776 +prometheus_http_response_size_bytes_count{handler="/config"} 13 +prometheus_http_response_size_bytes_bucket{handler="/consoles/*filepath",le="100"} 17 +prometheus_http_response_size_bytes_bucket{handler="/consoles/*filepath",le="1000"} 17 +prometheus_http_response_size_bytes_bucket{handler="/consoles/*filepath",le="10000"} 31 +prometheus_http_response_size_bytes_bucket{handler="/consoles/*filepath",le="100000"} 34 +prometheus_http_response_size_bytes_bucket{handler="/consoles/*filepath",le="1e+06"} 34 +prometheus_http_response_size_bytes_bucket{handler="/consoles/*filepath",le="1e+07"} 34 +prometheus_http_response_size_bytes_bucket{handler="/consoles/*filepath",le="1e+08"} 34 +prometheus_http_response_size_bytes_bucket{handler="/consoles/*filepath",le="1e+09"} 34 +prometheus_http_response_size_bytes_bucket{handler="/consoles/*filepath",le="+Inf"} 34 +prometheus_http_response_size_bytes_sum{handler="/consoles/*filepath"} 175556 +prometheus_http_response_size_bytes_count{handler="/consoles/*filepath"} 34 +prometheus_http_response_size_bytes_bucket{handler="/debug/*subpath",le="100"} 1 +prometheus_http_response_size_bytes_bucket{handler="/debug/*subpath",le="1000"} 1 +prometheus_http_response_size_bytes_bucket{handler="/debug/*subpath",le="10000"} 4 +prometheus_http_response_size_bytes_bucket{handler="/debug/*subpath",le="100000"} 4 +prometheus_http_response_size_bytes_bucket{handler="/debug/*subpath",le="1e+06"} 4 +prometheus_http_response_size_bytes_bucket{handler="/debug/*subpath",le="1e+07"} 4 +prometheus_http_response_size_bytes_bucket{handler="/debug/*subpath",le="1e+08"} 4 +prometheus_http_response_size_bytes_bucket{handler="/debug/*subpath",le="1e+09"} 4 +prometheus_http_response_size_bytes_bucket{handler="/debug/*subpath",le="+Inf"} 4 +prometheus_http_response_size_bytes_sum{handler="/debug/*subpath"} 7062 +prometheus_http_response_size_bytes_count{handler="/debug/*subpath"} 4 +prometheus_http_response_size_bytes_bucket{handler="/favicon.ico",le="100"} 177 +prometheus_http_response_size_bytes_bucket{handler="/favicon.ico",le="1000"} 177 +prometheus_http_response_size_bytes_bucket{handler="/favicon.ico",le="10000"} 177 +prometheus_http_response_size_bytes_bucket{handler="/favicon.ico",le="100000"} 177 +prometheus_http_response_size_bytes_bucket{handler="/favicon.ico",le="1e+06"} 177 +prometheus_http_response_size_bytes_bucket{handler="/favicon.ico",le="1e+07"} 177 +prometheus_http_response_size_bytes_bucket{handler="/favicon.ico",le="1e+08"} 177 +prometheus_http_response_size_bytes_bucket{handler="/favicon.ico",le="1e+09"} 177 +prometheus_http_response_size_bytes_bucket{handler="/favicon.ico",le="+Inf"} 177 +prometheus_http_response_size_bytes_sum{handler="/favicon.ico"} 3363 +prometheus_http_response_size_bytes_count{handler="/favicon.ico"} 177 +prometheus_http_response_size_bytes_bucket{handler="/favicon.svg",le="100"} 1 +prometheus_http_response_size_bytes_bucket{handler="/favicon.svg",le="1000"} 1 +prometheus_http_response_size_bytes_bucket{handler="/favicon.svg",le="10000"} 770 +prometheus_http_response_size_bytes_bucket{handler="/favicon.svg",le="100000"} 770 +prometheus_http_response_size_bytes_bucket{handler="/favicon.svg",le="1e+06"} 770 +prometheus_http_response_size_bytes_bucket{handler="/favicon.svg",le="1e+07"} 770 +prometheus_http_response_size_bytes_bucket{handler="/favicon.svg",le="1e+08"} 770 +prometheus_http_response_size_bytes_bucket{handler="/favicon.svg",le="1e+09"} 770 +prometheus_http_response_size_bytes_bucket{handler="/favicon.svg",le="+Inf"} 770 +prometheus_http_response_size_bytes_sum{handler="/favicon.svg"} 2.135541e+06 +prometheus_http_response_size_bytes_count{handler="/favicon.svg"} 770 +prometheus_http_response_size_bytes_bucket{handler="/federate",le="100"} 2 +prometheus_http_response_size_bytes_bucket{handler="/federate",le="1000"} 2 +prometheus_http_response_size_bytes_bucket{handler="/federate",le="10000"} 2 +prometheus_http_response_size_bytes_bucket{handler="/federate",le="100000"} 4 +prometheus_http_response_size_bytes_bucket{handler="/federate",le="1e+06"} 4 +prometheus_http_response_size_bytes_bucket{handler="/federate",le="1e+07"} 4 +prometheus_http_response_size_bytes_bucket{handler="/federate",le="1e+08"} 4 +prometheus_http_response_size_bytes_bucket{handler="/federate",le="1e+09"} 4 +prometheus_http_response_size_bytes_bucket{handler="/federate",le="+Inf"} 4 +prometheus_http_response_size_bytes_sum{handler="/federate"} 28439 +prometheus_http_response_size_bytes_count{handler="/federate"} 4 +prometheus_http_response_size_bytes_bucket{handler="/flags",le="100"} 0 +prometheus_http_response_size_bytes_bucket{handler="/flags",le="1000"} 0 +prometheus_http_response_size_bytes_bucket{handler="/flags",le="10000"} 9 +prometheus_http_response_size_bytes_bucket{handler="/flags",le="100000"} 9 +prometheus_http_response_size_bytes_bucket{handler="/flags",le="1e+06"} 9 +prometheus_http_response_size_bytes_bucket{handler="/flags",le="1e+07"} 9 +prometheus_http_response_size_bytes_bucket{handler="/flags",le="1e+08"} 9 +prometheus_http_response_size_bytes_bucket{handler="/flags",le="1e+09"} 9 +prometheus_http_response_size_bytes_bucket{handler="/flags",le="+Inf"} 9 +prometheus_http_response_size_bytes_sum{handler="/flags"} 15768 +prometheus_http_response_size_bytes_count{handler="/flags"} 9 +prometheus_http_response_size_bytes_bucket{handler="/graph",le="100"} 130 +prometheus_http_response_size_bytes_bucket{handler="/graph",le="1000"} 744 +prometheus_http_response_size_bytes_bucket{handler="/graph",le="10000"} 751 +prometheus_http_response_size_bytes_bucket{handler="/graph",le="100000"} 751 +prometheus_http_response_size_bytes_bucket{handler="/graph",le="1e+06"} 751 +prometheus_http_response_size_bytes_bucket{handler="/graph",le="1e+07"} 751 +prometheus_http_response_size_bytes_bucket{handler="/graph",le="1e+08"} 751 +prometheus_http_response_size_bytes_bucket{handler="/graph",le="1e+09"} 751 +prometheus_http_response_size_bytes_bucket{handler="/graph",le="+Inf"} 751 +prometheus_http_response_size_bytes_sum{handler="/graph"} 164775 +prometheus_http_response_size_bytes_count{handler="/graph"} 751 +prometheus_http_response_size_bytes_bucket{handler="/metrics",le="100"} 0 +prometheus_http_response_size_bytes_bucket{handler="/metrics",le="1000"} 0 +prometheus_http_response_size_bytes_bucket{handler="/metrics",le="10000"} 0 +prometheus_http_response_size_bytes_bucket{handler="/metrics",le="100000"} 4.059088e+06 +prometheus_http_response_size_bytes_bucket{handler="/metrics",le="1e+06"} 4.059092e+06 +prometheus_http_response_size_bytes_bucket{handler="/metrics",le="1e+07"} 4.059092e+06 +prometheus_http_response_size_bytes_bucket{handler="/metrics",le="1e+08"} 4.059092e+06 +prometheus_http_response_size_bytes_bucket{handler="/metrics",le="1e+09"} 4.059092e+06 +prometheus_http_response_size_bytes_bucket{handler="/metrics",le="+Inf"} 4.059092e+06 +prometheus_http_response_size_bytes_sum{handler="/metrics"} 8.2515439707e+10 +prometheus_http_response_size_bytes_count{handler="/metrics"} 4.059092e+06 +prometheus_http_response_size_bytes_bucket{handler="/query",le="100"} 0 +prometheus_http_response_size_bytes_bucket{handler="/query",le="1000"} 0 +prometheus_http_response_size_bytes_bucket{handler="/query",le="10000"} 1774 +prometheus_http_response_size_bytes_bucket{handler="/query",le="100000"} 1774 +prometheus_http_response_size_bytes_bucket{handler="/query",le="1e+06"} 1774 +prometheus_http_response_size_bytes_bucket{handler="/query",le="1e+07"} 1774 +prometheus_http_response_size_bytes_bucket{handler="/query",le="1e+08"} 1774 +prometheus_http_response_size_bytes_bucket{handler="/query",le="1e+09"} 1774 +prometheus_http_response_size_bytes_bucket{handler="/query",le="+Inf"} 1774 +prometheus_http_response_size_bytes_sum{handler="/query"} 3.108048e+06 +prometheus_http_response_size_bytes_count{handler="/query"} 1774 +prometheus_http_response_size_bytes_bucket{handler="/rules",le="100"} 0 +prometheus_http_response_size_bytes_bucket{handler="/rules",le="1000"} 0 +prometheus_http_response_size_bytes_bucket{handler="/rules",le="10000"} 8673 +prometheus_http_response_size_bytes_bucket{handler="/rules",le="100000"} 8673 +prometheus_http_response_size_bytes_bucket{handler="/rules",le="1e+06"} 8673 +prometheus_http_response_size_bytes_bucket{handler="/rules",le="1e+07"} 8673 +prometheus_http_response_size_bytes_bucket{handler="/rules",le="1e+08"} 8673 +prometheus_http_response_size_bytes_bucket{handler="/rules",le="1e+09"} 8673 +prometheus_http_response_size_bytes_bucket{handler="/rules",le="+Inf"} 8673 +prometheus_http_response_size_bytes_sum{handler="/rules"} 1.5195096e+07 +prometheus_http_response_size_bytes_count{handler="/rules"} 8673 +prometheus_http_response_size_bytes_bucket{handler="/service-discovery",le="100"} 0 +prometheus_http_response_size_bytes_bucket{handler="/service-discovery",le="1000"} 0 +prometheus_http_response_size_bytes_bucket{handler="/service-discovery",le="10000"} 20 +prometheus_http_response_size_bytes_bucket{handler="/service-discovery",le="100000"} 20 +prometheus_http_response_size_bytes_bucket{handler="/service-discovery",le="1e+06"} 20 +prometheus_http_response_size_bytes_bucket{handler="/service-discovery",le="1e+07"} 20 +prometheus_http_response_size_bytes_bucket{handler="/service-discovery",le="1e+08"} 20 +prometheus_http_response_size_bytes_bucket{handler="/service-discovery",le="1e+09"} 20 +prometheus_http_response_size_bytes_bucket{handler="/service-discovery",le="+Inf"} 20 +prometheus_http_response_size_bytes_sum{handler="/service-discovery"} 35040 +prometheus_http_response_size_bytes_count{handler="/service-discovery"} 20 +prometheus_http_response_size_bytes_bucket{handler="/status",le="100"} 0 +prometheus_http_response_size_bytes_bucket{handler="/status",le="1000"} 0 +prometheus_http_response_size_bytes_bucket{handler="/status",le="10000"} 46 +prometheus_http_response_size_bytes_bucket{handler="/status",le="100000"} 46 +prometheus_http_response_size_bytes_bucket{handler="/status",le="1e+06"} 46 +prometheus_http_response_size_bytes_bucket{handler="/status",le="1e+07"} 46 +prometheus_http_response_size_bytes_bucket{handler="/status",le="1e+08"} 46 +prometheus_http_response_size_bytes_bucket{handler="/status",le="1e+09"} 46 +prometheus_http_response_size_bytes_bucket{handler="/status",le="+Inf"} 46 +prometheus_http_response_size_bytes_sum{handler="/status"} 80592 +prometheus_http_response_size_bytes_count{handler="/status"} 46 +prometheus_http_response_size_bytes_bucket{handler="/targets",le="100"} 0 +prometheus_http_response_size_bytes_bucket{handler="/targets",le="1000"} 0 +prometheus_http_response_size_bytes_bucket{handler="/targets",le="10000"} 39 +prometheus_http_response_size_bytes_bucket{handler="/targets",le="100000"} 39 +prometheus_http_response_size_bytes_bucket{handler="/targets",le="1e+06"} 39 +prometheus_http_response_size_bytes_bucket{handler="/targets",le="1e+07"} 39 +prometheus_http_response_size_bytes_bucket{handler="/targets",le="1e+08"} 39 +prometheus_http_response_size_bytes_bucket{handler="/targets",le="1e+09"} 39 +prometheus_http_response_size_bytes_bucket{handler="/targets",le="+Inf"} 39 +prometheus_http_response_size_bytes_sum{handler="/targets"} 68328 +prometheus_http_response_size_bytes_count{handler="/targets"} 39 +prometheus_http_response_size_bytes_bucket{handler="/tsdb-status",le="100"} 0 +prometheus_http_response_size_bytes_bucket{handler="/tsdb-status",le="1000"} 0 +prometheus_http_response_size_bytes_bucket{handler="/tsdb-status",le="10000"} 49 +prometheus_http_response_size_bytes_bucket{handler="/tsdb-status",le="100000"} 49 +prometheus_http_response_size_bytes_bucket{handler="/tsdb-status",le="1e+06"} 49 +prometheus_http_response_size_bytes_bucket{handler="/tsdb-status",le="1e+07"} 49 +prometheus_http_response_size_bytes_bucket{handler="/tsdb-status",le="1e+08"} 49 +prometheus_http_response_size_bytes_bucket{handler="/tsdb-status",le="1e+09"} 49 +prometheus_http_response_size_bytes_bucket{handler="/tsdb-status",le="+Inf"} 49 +prometheus_http_response_size_bytes_sum{handler="/tsdb-status"} 85848 +prometheus_http_response_size_bytes_count{handler="/tsdb-status"} 49 +prometheus_notifications_alertmanagers_discovered 1 +prometheus_notifications_dropped_total 0 +prometheus_notifications_errors_total{alertmanager="http://demo.do.prometheus.io:9093/api/v2/alerts"} 0 +prometheus_notifications_latency_seconds{alertmanager="http://demo.do.prometheus.io:9093/api/v2/alerts",quantile="0.5"} 0.001566044 +prometheus_notifications_latency_seconds{alertmanager="http://demo.do.prometheus.io:9093/api/v2/alerts",quantile="0.9"} 0.003927931 +prometheus_notifications_latency_seconds{alertmanager="http://demo.do.prometheus.io:9093/api/v2/alerts",quantile="0.99"} 0.013928135 +prometheus_notifications_latency_seconds_sum{alertmanager="http://demo.do.prometheus.io:9093/api/v2/alerts"} 194.15032606200046 +prometheus_notifications_latency_seconds_count{alertmanager="http://demo.do.prometheus.io:9093/api/v2/alerts"} 75180 +prometheus_notifications_queue_capacity 10000 +prometheus_notifications_queue_length 0 +prometheus_notifications_sent_total{alertmanager="http://demo.do.prometheus.io:9093/api/v2/alerts"} 141616 +prometheus_ready 1 +prometheus_remote_storage_exemplars_in_total 4.959738e+06 +prometheus_remote_storage_highest_timestamp_in_seconds 1.738949375e+09 +prometheus_remote_storage_histograms_in_total 0 +prometheus_remote_storage_samples_in_total 6.00447296e+08 +prometheus_remote_storage_string_interner_zero_reference_releases_total 0 +prometheus_rule_evaluation_duration_seconds{quantile="0.5"} 0.000214623 +prometheus_rule_evaluation_duration_seconds{quantile="0.9"} 0.001456135 +prometheus_rule_evaluation_duration_seconds{quantile="0.99"} 0.008111814 +prometheus_rule_evaluation_duration_seconds_sum 5209.704794862625 +prometheus_rule_evaluation_duration_seconds_count 7.203456e+06 +prometheus_rule_evaluation_failures_total{rule_group="/etc/prometheus/rules/ansible_managed.rules;ansible managed alert rules"} 0 +prometheus_rule_evaluation_failures_total{rule_group="/etc/prometheus/rules/ansible_managed.yml;ansible managed alert rules"} 0 +prometheus_rule_evaluation_failures_total{rule_group="/etc/prometheus/rules/node_alerts.rules;node-exporter"} 0 +prometheus_rule_evaluation_failures_total{rule_group="/etc/prometheus/rules/node_alerts.yaml;node-exporter"} 0 +prometheus_rule_evaluation_failures_total{rule_group="/etc/prometheus/rules/node_rules.rules;node-exporter.rules"} 0 +prometheus_rule_evaluation_failures_total{rule_group="/etc/prometheus/rules/node_rules.yaml;node-exporter.rules"} 0 +prometheus_rule_evaluation_failures_total{rule_group="/etc/prometheus/rules/prometheus_alerts.rules;prometheus"} 0 +prometheus_rule_evaluation_failures_total{rule_group="/etc/prometheus/rules/prometheus_alerts.yaml;prometheus"} 0 +prometheus_rule_evaluations_total{rule_group="/etc/prometheus/rules/ansible_managed.rules;ansible managed alert rules"} 118092 +prometheus_rule_evaluations_total{rule_group="/etc/prometheus/rules/ansible_managed.yml;ansible managed alert rules"} 118090 +prometheus_rule_evaluations_total{rule_group="/etc/prometheus/rules/node_alerts.rules;node-exporter"} 1.4761e+06 +prometheus_rule_evaluations_total{rule_group="/etc/prometheus/rules/node_alerts.yaml;node-exporter"} 1.476125e+06 +prometheus_rule_evaluations_total{rule_group="/etc/prometheus/rules/node_rules.rules;node-exporter.rules"} 649495 +prometheus_rule_evaluations_total{rule_group="/etc/prometheus/rules/node_rules.yaml;node-exporter.rules"} 649484 +prometheus_rule_evaluations_total{rule_group="/etc/prometheus/rules/prometheus_alerts.rules;prometheus"} 1.358035e+06 +prometheus_rule_evaluations_total{rule_group="/etc/prometheus/rules/prometheus_alerts.yaml;prometheus"} 1.358035e+06 +prometheus_rule_group_duration_seconds{quantile="0.01"} 0.000735928 +prometheus_rule_group_duration_seconds{quantile="0.05"} 0.000818857 +prometheus_rule_group_duration_seconds{quantile="0.5"} 0.004852081 +prometheus_rule_group_duration_seconds{quantile="0.9"} 0.022897759 +prometheus_rule_group_duration_seconds{quantile="0.99"} 0.069327797 +prometheus_rule_group_duration_seconds_sum 5335.451440133042 +prometheus_rule_group_duration_seconds_count 472359 +prometheus_rule_group_interval_seconds{rule_group="/etc/prometheus/rules/ansible_managed.rules;ansible managed alert rules"} 15 +prometheus_rule_group_interval_seconds{rule_group="/etc/prometheus/rules/ansible_managed.yml;ansible managed alert rules"} 15 +prometheus_rule_group_interval_seconds{rule_group="/etc/prometheus/rules/node_alerts.rules;node-exporter"} 15 +prometheus_rule_group_interval_seconds{rule_group="/etc/prometheus/rules/node_alerts.yaml;node-exporter"} 15 +prometheus_rule_group_interval_seconds{rule_group="/etc/prometheus/rules/node_rules.rules;node-exporter.rules"} 15 +prometheus_rule_group_interval_seconds{rule_group="/etc/prometheus/rules/node_rules.yaml;node-exporter.rules"} 15 +prometheus_rule_group_interval_seconds{rule_group="/etc/prometheus/rules/prometheus_alerts.rules;prometheus"} 15 +prometheus_rule_group_interval_seconds{rule_group="/etc/prometheus/rules/prometheus_alerts.yaml;prometheus"} 15 +prometheus_rule_group_iterations_missed_total{rule_group="/etc/prometheus/rules/ansible_managed.rules;ansible managed alert rules"} 0 +prometheus_rule_group_iterations_missed_total{rule_group="/etc/prometheus/rules/ansible_managed.yml;ansible managed alert rules"} 0 +prometheus_rule_group_iterations_missed_total{rule_group="/etc/prometheus/rules/node_alerts.rules;node-exporter"} 1 +prometheus_rule_group_iterations_missed_total{rule_group="/etc/prometheus/rules/node_alerts.yaml;node-exporter"} 0 +prometheus_rule_group_iterations_missed_total{rule_group="/etc/prometheus/rules/node_rules.rules;node-exporter.rules"} 0 +prometheus_rule_group_iterations_missed_total{rule_group="/etc/prometheus/rules/node_rules.yaml;node-exporter.rules"} 1 +prometheus_rule_group_iterations_missed_total{rule_group="/etc/prometheus/rules/prometheus_alerts.rules;prometheus"} 0 +prometheus_rule_group_iterations_missed_total{rule_group="/etc/prometheus/rules/prometheus_alerts.yaml;prometheus"} 0 +prometheus_rule_group_iterations_total{rule_group="/etc/prometheus/rules/ansible_managed.rules;ansible managed alert rules"} 59046 +prometheus_rule_group_iterations_total{rule_group="/etc/prometheus/rules/ansible_managed.yml;ansible managed alert rules"} 59045 +prometheus_rule_group_iterations_total{rule_group="/etc/prometheus/rules/node_alerts.rules;node-exporter"} 59045 +prometheus_rule_group_iterations_total{rule_group="/etc/prometheus/rules/node_alerts.yaml;node-exporter"} 59045 +prometheus_rule_group_iterations_total{rule_group="/etc/prometheus/rules/node_rules.rules;node-exporter.rules"} 59045 +prometheus_rule_group_iterations_total{rule_group="/etc/prometheus/rules/node_rules.yaml;node-exporter.rules"} 59045 +prometheus_rule_group_iterations_total{rule_group="/etc/prometheus/rules/prometheus_alerts.rules;prometheus"} 59045 +prometheus_rule_group_iterations_total{rule_group="/etc/prometheus/rules/prometheus_alerts.yaml;prometheus"} 59045 +prometheus_rule_group_last_duration_seconds{rule_group="/etc/prometheus/rules/ansible_managed.rules;ansible managed alert rules"} 0.000754015 +prometheus_rule_group_last_duration_seconds{rule_group="/etc/prometheus/rules/ansible_managed.yml;ansible managed alert rules"} 0.001104624 +prometheus_rule_group_last_duration_seconds{rule_group="/etc/prometheus/rules/node_alerts.rules;node-exporter"} 0.022040842 +prometheus_rule_group_last_duration_seconds{rule_group="/etc/prometheus/rules/node_alerts.yaml;node-exporter"} 0.048928087 +prometheus_rule_group_last_duration_seconds{rule_group="/etc/prometheus/rules/node_rules.rules;node-exporter.rules"} 0.002766703 +prometheus_rule_group_last_duration_seconds{rule_group="/etc/prometheus/rules/node_rules.yaml;node-exporter.rules"} 0.002218466 +prometheus_rule_group_last_duration_seconds{rule_group="/etc/prometheus/rules/prometheus_alerts.rules;prometheus"} 0.010245447 +prometheus_rule_group_last_duration_seconds{rule_group="/etc/prometheus/rules/prometheus_alerts.yaml;prometheus"} 0.009232393 +prometheus_rule_group_last_evaluation_samples{rule_group="/etc/prometheus/rules/ansible_managed.rules;ansible managed alert rules"} 2 +prometheus_rule_group_last_evaluation_samples{rule_group="/etc/prometheus/rules/ansible_managed.yml;ansible managed alert rules"} 2 +prometheus_rule_group_last_evaluation_samples{rule_group="/etc/prometheus/rules/node_alerts.rules;node-exporter"} 10 +prometheus_rule_group_last_evaluation_samples{rule_group="/etc/prometheus/rules/node_alerts.yaml;node-exporter"} 10 +prometheus_rule_group_last_evaluation_samples{rule_group="/etc/prometheus/rules/node_rules.rules;node-exporter.rules"} 11 +prometheus_rule_group_last_evaluation_samples{rule_group="/etc/prometheus/rules/node_rules.yaml;node-exporter.rules"} 11 +prometheus_rule_group_last_evaluation_samples{rule_group="/etc/prometheus/rules/prometheus_alerts.rules;prometheus"} 0 +prometheus_rule_group_last_evaluation_samples{rule_group="/etc/prometheus/rules/prometheus_alerts.yaml;prometheus"} 0 +prometheus_rule_group_last_evaluation_timestamp_seconds{rule_group="/etc/prometheus/rules/ansible_managed.rules;ansible managed alert rules"} 1.738949373753179e+09 +prometheus_rule_group_last_evaluation_timestamp_seconds{rule_group="/etc/prometheus/rules/ansible_managed.yml;ansible managed alert rules"} 1.738949366900781e+09 +prometheus_rule_group_last_evaluation_timestamp_seconds{rule_group="/etc/prometheus/rules/node_alerts.rules;node-exporter"} 1.7389493632087085e+09 +prometheus_rule_group_last_evaluation_timestamp_seconds{rule_group="/etc/prometheus/rules/node_alerts.yaml;node-exporter"} 1.7389493666743164e+09 +prometheus_rule_group_last_evaluation_timestamp_seconds{rule_group="/etc/prometheus/rules/node_rules.rules;node-exporter.rules"} 1.7389493675395968e+09 +prometheus_rule_group_last_evaluation_timestamp_seconds{rule_group="/etc/prometheus/rules/node_rules.yaml;node-exporter.rules"} 1.7389493623381927e+09 +prometheus_rule_group_last_evaluation_timestamp_seconds{rule_group="/etc/prometheus/rules/prometheus_alerts.rules;prometheus"} 1.738949366856423e+09 +prometheus_rule_group_last_evaluation_timestamp_seconds{rule_group="/etc/prometheus/rules/prometheus_alerts.yaml;prometheus"} 1.738949369917499e+09 +prometheus_rule_group_last_restore_duration_seconds{rule_group="/etc/prometheus/rules/ansible_managed.rules;ansible managed alert rules"} 0.002032625 +prometheus_rule_group_last_restore_duration_seconds{rule_group="/etc/prometheus/rules/ansible_managed.yml;ansible managed alert rules"} 9.1853e-05 +prometheus_rule_group_last_restore_duration_seconds{rule_group="/etc/prometheus/rules/node_alerts.rules;node-exporter"} 0.000166088 +prometheus_rule_group_last_restore_duration_seconds{rule_group="/etc/prometheus/rules/node_alerts.yaml;node-exporter"} 0.000108127 +prometheus_rule_group_last_restore_duration_seconds{rule_group="/etc/prometheus/rules/node_rules.rules;node-exporter.rules"} 6.408e-06 +prometheus_rule_group_last_restore_duration_seconds{rule_group="/etc/prometheus/rules/node_rules.yaml;node-exporter.rules"} 2.621e-06 +prometheus_rule_group_last_restore_duration_seconds{rule_group="/etc/prometheus/rules/prometheus_alerts.rules;prometheus"} 8.4979e-05 +prometheus_rule_group_last_restore_duration_seconds{rule_group="/etc/prometheus/rules/prometheus_alerts.yaml;prometheus"} 0.000104444 +prometheus_rule_group_last_rule_duration_sum_seconds{rule_group="/etc/prometheus/rules/ansible_managed.rules;ansible managed alert rules"} 0.000685176 +prometheus_rule_group_last_rule_duration_sum_seconds{rule_group="/etc/prometheus/rules/ansible_managed.yml;ansible managed alert rules"} 0.001035099 +prometheus_rule_group_last_rule_duration_sum_seconds{rule_group="/etc/prometheus/rules/node_alerts.rules;node-exporter"} 0.021769662 +prometheus_rule_group_last_rule_duration_sum_seconds{rule_group="/etc/prometheus/rules/node_alerts.yaml;node-exporter"} 0.042017156 +prometheus_rule_group_last_rule_duration_sum_seconds{rule_group="/etc/prometheus/rules/node_rules.rules;node-exporter.rules"} 0.00260535 +prometheus_rule_group_last_rule_duration_sum_seconds{rule_group="/etc/prometheus/rules/node_rules.yaml;node-exporter.rules"} 0.002069356 +prometheus_rule_group_last_rule_duration_sum_seconds{rule_group="/etc/prometheus/rules/prometheus_alerts.rules;prometheus"} 0.009905481 +prometheus_rule_group_last_rule_duration_sum_seconds{rule_group="/etc/prometheus/rules/prometheus_alerts.yaml;prometheus"} 0.009056893 +prometheus_rule_group_rules{rule_group="/etc/prometheus/rules/ansible_managed.rules;ansible managed alert rules"} 2 +prometheus_rule_group_rules{rule_group="/etc/prometheus/rules/ansible_managed.yml;ansible managed alert rules"} 2 +prometheus_rule_group_rules{rule_group="/etc/prometheus/rules/node_alerts.rules;node-exporter"} 25 +prometheus_rule_group_rules{rule_group="/etc/prometheus/rules/node_alerts.yaml;node-exporter"} 25 +prometheus_rule_group_rules{rule_group="/etc/prometheus/rules/node_rules.rules;node-exporter.rules"} 11 +prometheus_rule_group_rules{rule_group="/etc/prometheus/rules/node_rules.yaml;node-exporter.rules"} 11 +prometheus_rule_group_rules{rule_group="/etc/prometheus/rules/prometheus_alerts.rules;prometheus"} 23 +prometheus_rule_group_rules{rule_group="/etc/prometheus/rules/prometheus_alerts.yaml;prometheus"} 23 +prometheus_sd_azure_cache_hit_total 0 +prometheus_sd_azure_failures_total 0 +prometheus_sd_consul_rpc_duration_seconds{call="service",endpoint="catalog",quantile="0.5"} NaN +prometheus_sd_consul_rpc_duration_seconds{call="service",endpoint="catalog",quantile="0.9"} NaN +prometheus_sd_consul_rpc_duration_seconds{call="service",endpoint="catalog",quantile="0.99"} NaN +prometheus_sd_consul_rpc_duration_seconds_sum{call="service",endpoint="catalog"} 0 +prometheus_sd_consul_rpc_duration_seconds_count{call="service",endpoint="catalog"} 0 +prometheus_sd_consul_rpc_duration_seconds{call="services",endpoint="catalog",quantile="0.5"} NaN +prometheus_sd_consul_rpc_duration_seconds{call="services",endpoint="catalog",quantile="0.9"} NaN +prometheus_sd_consul_rpc_duration_seconds{call="services",endpoint="catalog",quantile="0.99"} NaN +prometheus_sd_consul_rpc_duration_seconds_sum{call="services",endpoint="catalog"} 0 +prometheus_sd_consul_rpc_duration_seconds_count{call="services",endpoint="catalog"} 0 +prometheus_sd_consul_rpc_failures_total 0 +prometheus_sd_discovered_targets{config="alertmanager",name="scrape"} 1 +prometheus_sd_discovered_targets{config="blackbox",name="scrape"} 1 +prometheus_sd_discovered_targets{config="caddy",name="scrape"} 1 +prometheus_sd_discovered_targets{config="cadvisor",name="scrape"} 1 +prometheus_sd_discovered_targets{config="config-0",name="notify"} 1 +prometheus_sd_discovered_targets{config="grafana",name="scrape"} 1 +prometheus_sd_discovered_targets{config="node",name="scrape"} 1 +prometheus_sd_discovered_targets{config="prometheus",name="scrape"} 1 +prometheus_sd_discovered_targets{config="random",name="scrape"} 4 +prometheus_sd_dns_lookup_failures_total 0 +prometheus_sd_dns_lookups_total 0 +prometheus_sd_failed_configs{name="notify"} 0 +prometheus_sd_failed_configs{name="scrape"} 0 +prometheus_sd_file_mtime_seconds{filename="/etc/prometheus/file_sd/alertmanager.yml"} 1.701295557e+09 +prometheus_sd_file_mtime_seconds{filename="/etc/prometheus/file_sd/cadvisor.yml"} 1.705433682e+09 +prometheus_sd_file_mtime_seconds{filename="/etc/prometheus/file_sd/node.yml"} 1.701295553e+09 +prometheus_sd_file_mtime_seconds{filename="/etc/prometheus/file_sd/random.yml"} 1.579263729e+09 +prometheus_sd_file_read_errors_total 0 +prometheus_sd_file_scan_duration_seconds{quantile="0.5"} 9.5384e-05 +prometheus_sd_file_scan_duration_seconds{quantile="0.9"} 0.000275679 +prometheus_sd_file_scan_duration_seconds{quantile="0.99"} 0.000275679 +prometheus_sd_file_scan_duration_seconds_sum 4.751347856999997 +prometheus_sd_file_scan_duration_seconds_count 11812 +prometheus_sd_file_watcher_errors_total 0 +prometheus_sd_http_failures_total 0 +prometheus_sd_kubernetes_events_total{event="add",role="endpoints"} 0 +prometheus_sd_kubernetes_events_total{event="add",role="endpointslice"} 0 +prometheus_sd_kubernetes_events_total{event="add",role="ingress"} 0 +prometheus_sd_kubernetes_events_total{event="add",role="node"} 0 +prometheus_sd_kubernetes_events_total{event="add",role="pod"} 0 +prometheus_sd_kubernetes_events_total{event="add",role="service"} 0 +prometheus_sd_kubernetes_events_total{event="delete",role="endpoints"} 0 +prometheus_sd_kubernetes_events_total{event="delete",role="endpointslice"} 0 +prometheus_sd_kubernetes_events_total{event="delete",role="ingress"} 0 +prometheus_sd_kubernetes_events_total{event="delete",role="node"} 0 +prometheus_sd_kubernetes_events_total{event="delete",role="pod"} 0 +prometheus_sd_kubernetes_events_total{event="delete",role="service"} 0 +prometheus_sd_kubernetes_events_total{event="update",role="endpoints"} 0 +prometheus_sd_kubernetes_events_total{event="update",role="endpointslice"} 0 +prometheus_sd_kubernetes_events_total{event="update",role="ingress"} 0 +prometheus_sd_kubernetes_events_total{event="update",role="node"} 0 +prometheus_sd_kubernetes_events_total{event="update",role="pod"} 0 +prometheus_sd_kubernetes_events_total{event="update",role="service"} 0 +prometheus_sd_kubernetes_failures_total 0 +prometheus_sd_kuma_fetch_duration_seconds{quantile="0.5"} NaN +prometheus_sd_kuma_fetch_duration_seconds{quantile="0.9"} NaN +prometheus_sd_kuma_fetch_duration_seconds{quantile="0.99"} NaN +prometheus_sd_kuma_fetch_duration_seconds_sum 0 +prometheus_sd_kuma_fetch_duration_seconds_count 0 +prometheus_sd_kuma_fetch_failures_total 0 +prometheus_sd_kuma_fetch_skipped_updates_total 0 +prometheus_sd_linode_failures_total 0 +prometheus_sd_nomad_failures_total 0 +prometheus_sd_received_updates_total{name="notify"} 2 +prometheus_sd_received_updates_total{name="scrape"} 11820 +prometheus_sd_updates_delayed_total{name="notify"} 0 +prometheus_sd_updates_delayed_total{name="scrape"} 0 +prometheus_sd_updates_total{name="notify"} 1 +prometheus_sd_updates_total{name="scrape"} 2953 +prometheus_target_interval_length_seconds{interval="15s",quantile="0.01"} 14.982591232 +prometheus_target_interval_length_seconds{interval="15s",quantile="0.05"} 14.997567414 +prometheus_target_interval_length_seconds{interval="15s",quantile="0.5"} 14.999977915 +prometheus_target_interval_length_seconds{interval="15s",quantile="0.9"} 15.000793403 +prometheus_target_interval_length_seconds{interval="15s",quantile="0.99"} 15.017607167 +prometheus_target_interval_length_seconds_sum{interval="15s"} 9.742237453453667e+06 +prometheus_target_interval_length_seconds_count{interval="15s"} 649482 +prometheus_target_metadata_cache_bytes{scrape_job="alertmanager"} 6817 +prometheus_target_metadata_cache_bytes{scrape_job="blackbox"} 661 +prometheus_target_metadata_cache_bytes{scrape_job="caddy"} 2365 +prometheus_target_metadata_cache_bytes{scrape_job="cadvisor"} 4547 +prometheus_target_metadata_cache_bytes{scrape_job="grafana"} 37608 +prometheus_target_metadata_cache_bytes{scrape_job="node"} 13900 +prometheus_target_metadata_cache_bytes{scrape_job="prometheus"} 20265 +prometheus_target_metadata_cache_bytes{scrape_job="random"} 792 +prometheus_target_metadata_cache_entries{scrape_job="alertmanager"} 86 +prometheus_target_metadata_cache_entries{scrape_job="blackbox"} 13 +prometheus_target_metadata_cache_entries{scrape_job="caddy"} 47 +prometheus_target_metadata_cache_entries{scrape_job="cadvisor"} 93 +prometheus_target_metadata_cache_entries{scrape_job="grafana"} 373 +prometheus_target_metadata_cache_entries{scrape_job="node"} 293 +prometheus_target_metadata_cache_entries{scrape_job="prometheus"} 237 +prometheus_target_metadata_cache_entries{scrape_job="random"} 16 +prometheus_target_scrape_pool_exceeded_label_limits_total 0 +prometheus_target_scrape_pool_exceeded_target_limit_total 0 +prometheus_target_scrape_pool_reloads_failed_total 0 +prometheus_target_scrape_pool_reloads_total 0 +prometheus_target_scrape_pool_symboltable_items{scrape_job="alertmanager"} 0 +prometheus_target_scrape_pool_symboltable_items{scrape_job="blackbox"} 0 +prometheus_target_scrape_pool_symboltable_items{scrape_job="caddy"} 0 +prometheus_target_scrape_pool_symboltable_items{scrape_job="cadvisor"} 0 +prometheus_target_scrape_pool_symboltable_items{scrape_job="grafana"} 0 +prometheus_target_scrape_pool_symboltable_items{scrape_job="node"} 0 +prometheus_target_scrape_pool_symboltable_items{scrape_job="prometheus"} 0 +prometheus_target_scrape_pool_symboltable_items{scrape_job="random"} 0 +prometheus_target_scrape_pool_sync_total{scrape_job="alertmanager"} 2953 +prometheus_target_scrape_pool_sync_total{scrape_job="blackbox"} 2953 +prometheus_target_scrape_pool_sync_total{scrape_job="caddy"} 2953 +prometheus_target_scrape_pool_sync_total{scrape_job="cadvisor"} 2953 +prometheus_target_scrape_pool_sync_total{scrape_job="grafana"} 2953 +prometheus_target_scrape_pool_sync_total{scrape_job="node"} 2953 +prometheus_target_scrape_pool_sync_total{scrape_job="prometheus"} 2953 +prometheus_target_scrape_pool_sync_total{scrape_job="random"} 2953 +prometheus_target_scrape_pool_target_limit{scrape_job="alertmanager"} 0 +prometheus_target_scrape_pool_target_limit{scrape_job="blackbox"} 0 +prometheus_target_scrape_pool_target_limit{scrape_job="caddy"} 0 +prometheus_target_scrape_pool_target_limit{scrape_job="cadvisor"} 0 +prometheus_target_scrape_pool_target_limit{scrape_job="grafana"} 0 +prometheus_target_scrape_pool_target_limit{scrape_job="node"} 0 +prometheus_target_scrape_pool_target_limit{scrape_job="prometheus"} 0 +prometheus_target_scrape_pool_target_limit{scrape_job="random"} 0 +prometheus_target_scrape_pool_targets{scrape_job="alertmanager"} 1 +prometheus_target_scrape_pool_targets{scrape_job="blackbox"} 1 +prometheus_target_scrape_pool_targets{scrape_job="caddy"} 1 +prometheus_target_scrape_pool_targets{scrape_job="cadvisor"} 1 +prometheus_target_scrape_pool_targets{scrape_job="grafana"} 1 +prometheus_target_scrape_pool_targets{scrape_job="node"} 1 +prometheus_target_scrape_pool_targets{scrape_job="prometheus"} 1 +prometheus_target_scrape_pool_targets{scrape_job="random"} 4 +prometheus_target_scrape_pools_failed_total 0 +prometheus_target_scrape_pools_total 8 +prometheus_target_scrapes_cache_flush_forced_total 0 +prometheus_target_scrapes_exceeded_body_size_limit_total 0 +prometheus_target_scrapes_exceeded_native_histogram_bucket_limit_total 0 +prometheus_target_scrapes_exceeded_sample_limit_total 0 +prometheus_target_scrapes_exemplar_out_of_order_total 0 +prometheus_target_scrapes_sample_duplicate_timestamp_total 0 +prometheus_target_scrapes_sample_out_of_bounds_total 0 +prometheus_target_scrapes_sample_out_of_order_total 455 +prometheus_target_sync_failed_total{scrape_job="alertmanager"} 0 +prometheus_target_sync_failed_total{scrape_job="blackbox"} 0 +prometheus_target_sync_failed_total{scrape_job="caddy"} 0 +prometheus_target_sync_failed_total{scrape_job="cadvisor"} 0 +prometheus_target_sync_failed_total{scrape_job="grafana"} 0 +prometheus_target_sync_failed_total{scrape_job="node"} 0 +prometheus_target_sync_failed_total{scrape_job="prometheus"} 0 +prometheus_target_sync_failed_total{scrape_job="random"} 0 +prometheus_target_sync_length_seconds{scrape_job="alertmanager",quantile="0.01"} 2.0522e-05 +prometheus_target_sync_length_seconds{scrape_job="alertmanager",quantile="0.05"} 2.0522e-05 +prometheus_target_sync_length_seconds{scrape_job="alertmanager",quantile="0.5"} 2.0522e-05 +prometheus_target_sync_length_seconds{scrape_job="alertmanager",quantile="0.9"} 0.000141485 +prometheus_target_sync_length_seconds{scrape_job="alertmanager",quantile="0.99"} 0.000141485 +prometheus_target_sync_length_seconds_sum{scrape_job="alertmanager"} 0.13103036 +prometheus_target_sync_length_seconds_count{scrape_job="alertmanager"} 2953 +prometheus_target_sync_length_seconds{scrape_job="blackbox",quantile="0.01"} 3.9252e-05 +prometheus_target_sync_length_seconds{scrape_job="blackbox",quantile="0.05"} 3.9252e-05 +prometheus_target_sync_length_seconds{scrape_job="blackbox",quantile="0.5"} 3.9252e-05 +prometheus_target_sync_length_seconds{scrape_job="blackbox",quantile="0.9"} 6.2134e-05 +prometheus_target_sync_length_seconds{scrape_job="blackbox",quantile="0.99"} 6.2134e-05 +prometheus_target_sync_length_seconds_sum{scrape_job="blackbox"} 0.6044201539999996 +prometheus_target_sync_length_seconds_count{scrape_job="blackbox"} 2953 +prometheus_target_sync_length_seconds{scrape_job="caddy",quantile="0.01"} 1.3759e-05 +prometheus_target_sync_length_seconds{scrape_job="caddy",quantile="0.05"} 1.3759e-05 +prometheus_target_sync_length_seconds{scrape_job="caddy",quantile="0.5"} 1.3759e-05 +prometheus_target_sync_length_seconds{scrape_job="caddy",quantile="0.9"} 7.8256e-05 +prometheus_target_sync_length_seconds{scrape_job="caddy",quantile="0.99"} 7.8256e-05 +prometheus_target_sync_length_seconds_sum{scrape_job="caddy"} 0.10369844599999971 +prometheus_target_sync_length_seconds_count{scrape_job="caddy"} 2953 +prometheus_target_sync_length_seconds{scrape_job="cadvisor",quantile="0.01"} 2.0452e-05 +prometheus_target_sync_length_seconds{scrape_job="cadvisor",quantile="0.05"} 2.0452e-05 +prometheus_target_sync_length_seconds{scrape_job="cadvisor",quantile="0.5"} 2.0452e-05 +prometheus_target_sync_length_seconds{scrape_job="cadvisor",quantile="0.9"} 4.2337e-05 +prometheus_target_sync_length_seconds{scrape_job="cadvisor",quantile="0.99"} 4.2337e-05 +prometheus_target_sync_length_seconds_sum{scrape_job="cadvisor"} 0.10489659999999998 +prometheus_target_sync_length_seconds_count{scrape_job="cadvisor"} 2953 +prometheus_target_sync_length_seconds{scrape_job="grafana",quantile="0.01"} 1.4995e-05 +prometheus_target_sync_length_seconds{scrape_job="grafana",quantile="0.05"} 1.4995e-05 +prometheus_target_sync_length_seconds{scrape_job="grafana",quantile="0.5"} 1.4995e-05 +prometheus_target_sync_length_seconds{scrape_job="grafana",quantile="0.9"} 1.7284e-05 +prometheus_target_sync_length_seconds{scrape_job="grafana",quantile="0.99"} 1.7284e-05 +prometheus_target_sync_length_seconds_sum{scrape_job="grafana"} 0.09031192700000017 +prometheus_target_sync_length_seconds_count{scrape_job="grafana"} 2953 +prometheus_target_sync_length_seconds{scrape_job="node",quantile="0.01"} 4.1607e-05 +prometheus_target_sync_length_seconds{scrape_job="node",quantile="0.05"} 4.1607e-05 +prometheus_target_sync_length_seconds{scrape_job="node",quantile="0.5"} 4.1607e-05 +prometheus_target_sync_length_seconds{scrape_job="node",quantile="0.9"} 7.416e-05 +prometheus_target_sync_length_seconds{scrape_job="node",quantile="0.99"} 7.416e-05 +prometheus_target_sync_length_seconds_sum{scrape_job="node"} 0.11539821299999993 +prometheus_target_sync_length_seconds_count{scrape_job="node"} 2953 +prometheus_target_sync_length_seconds{scrape_job="prometheus",quantile="0.01"} 1.5564e-05 +prometheus_target_sync_length_seconds{scrape_job="prometheus",quantile="0.05"} 1.5564e-05 +prometheus_target_sync_length_seconds{scrape_job="prometheus",quantile="0.5"} 1.5564e-05 +prometheus_target_sync_length_seconds{scrape_job="prometheus",quantile="0.9"} 1.961e-05 +prometheus_target_sync_length_seconds{scrape_job="prometheus",quantile="0.99"} 1.961e-05 +prometheus_target_sync_length_seconds_sum{scrape_job="prometheus"} 0.10655758600000016 +prometheus_target_sync_length_seconds_count{scrape_job="prometheus"} 2953 +prometheus_target_sync_length_seconds{scrape_job="random",quantile="0.01"} 4.1299e-05 +prometheus_target_sync_length_seconds{scrape_job="random",quantile="0.05"} 4.1299e-05 +prometheus_target_sync_length_seconds{scrape_job="random",quantile="0.5"} 4.1299e-05 +prometheus_target_sync_length_seconds{scrape_job="random",quantile="0.9"} 4.8586e-05 +prometheus_target_sync_length_seconds{scrape_job="random",quantile="0.99"} 4.8586e-05 +prometheus_target_sync_length_seconds_sum{scrape_job="random"} 0.20406449899999993 +prometheus_target_sync_length_seconds_count{scrape_job="random"} 2953 +prometheus_template_text_expansion_failures_total 0 +prometheus_template_text_expansions_total 2.126277e+06 +prometheus_treecache_watcher_goroutines 0 +prometheus_treecache_zookeeper_failures_total 0 +prometheus_tsdb_blocks_loaded 17 +prometheus_tsdb_checkpoint_creations_failed_total 0 +prometheus_tsdb_checkpoint_creations_total 62 +prometheus_tsdb_checkpoint_deletions_failed_total 0 +prometheus_tsdb_checkpoint_deletions_total 62 +prometheus_tsdb_clean_start 1 +prometheus_tsdb_compaction_chunk_range_seconds_bucket{le="100"} 86 +prometheus_tsdb_compaction_chunk_range_seconds_bucket{le="400"} 86 +prometheus_tsdb_compaction_chunk_range_seconds_bucket{le="1600"} 86 +prometheus_tsdb_compaction_chunk_range_seconds_bucket{le="6400"} 86 +prometheus_tsdb_compaction_chunk_range_seconds_bucket{le="25600"} 676 +prometheus_tsdb_compaction_chunk_range_seconds_bucket{le="102400"} 1555 +prometheus_tsdb_compaction_chunk_range_seconds_bucket{le="409600"} 2379 +prometheus_tsdb_compaction_chunk_range_seconds_bucket{le="1.6384e+06"} 34068 +prometheus_tsdb_compaction_chunk_range_seconds_bucket{le="6.5536e+06"} 4.819758e+06 +prometheus_tsdb_compaction_chunk_range_seconds_bucket{le="2.62144e+07"} 4.861956e+06 +prometheus_tsdb_compaction_chunk_range_seconds_bucket{le="+Inf"} 4.861956e+06 +prometheus_tsdb_compaction_chunk_range_seconds_sum 8.917524773366e+12 +prometheus_tsdb_compaction_chunk_range_seconds_count 4.861956e+06 +prometheus_tsdb_compaction_chunk_samples_bucket{le="4"} 1407 +prometheus_tsdb_compaction_chunk_samples_bucket{le="6"} 1544 +prometheus_tsdb_compaction_chunk_samples_bucket{le="9"} 1881 +prometheus_tsdb_compaction_chunk_samples_bucket{le="13.5"} 2065 +prometheus_tsdb_compaction_chunk_samples_bucket{le="20.25"} 2782 +prometheus_tsdb_compaction_chunk_samples_bucket{le="30.375"} 4342 +prometheus_tsdb_compaction_chunk_samples_bucket{le="45.5625"} 6180 +prometheus_tsdb_compaction_chunk_samples_bucket{le="68.34375"} 11518 +prometheus_tsdb_compaction_chunk_samples_bucket{le="102.515625"} 13890 +prometheus_tsdb_compaction_chunk_samples_bucket{le="153.7734375"} 4.810155e+06 +prometheus_tsdb_compaction_chunk_samples_bucket{le="230.66015625"} 4.86058e+06 +prometheus_tsdb_compaction_chunk_samples_bucket{le="345.990234375"} 4.861956e+06 +prometheus_tsdb_compaction_chunk_samples_bucket{le="+Inf"} 4.861956e+06 +prometheus_tsdb_compaction_chunk_samples_sum 5.86000708e+08 +prometheus_tsdb_compaction_chunk_samples_count 4.861956e+06 +prometheus_tsdb_compaction_chunk_size_bytes_bucket{le="32"} 1233 +prometheus_tsdb_compaction_chunk_size_bytes_bucket{le="48"} 156238 +prometheus_tsdb_compaction_chunk_size_bytes_bucket{le="72"} 2.006456e+06 +prometheus_tsdb_compaction_chunk_size_bytes_bucket{le="108"} 3.568405e+06 +prometheus_tsdb_compaction_chunk_size_bytes_bucket{le="162"} 3.835144e+06 +prometheus_tsdb_compaction_chunk_size_bytes_bucket{le="243"} 4.034591e+06 +prometheus_tsdb_compaction_chunk_size_bytes_bucket{le="364.5"} 4.505646e+06 +prometheus_tsdb_compaction_chunk_size_bytes_bucket{le="546.75"} 4.69694e+06 +prometheus_tsdb_compaction_chunk_size_bytes_bucket{le="820.125"} 4.78551e+06 +prometheus_tsdb_compaction_chunk_size_bytes_bucket{le="1230.1875"} 4.861956e+06 +prometheus_tsdb_compaction_chunk_size_bytes_bucket{le="1845.28125"} 4.861956e+06 +prometheus_tsdb_compaction_chunk_size_bytes_bucket{le="2767.921875"} 4.861956e+06 +prometheus_tsdb_compaction_chunk_size_bytes_bucket{le="+Inf"} 4.861956e+06 +prometheus_tsdb_compaction_chunk_size_bytes_sum 6.81852972e+08 +prometheus_tsdb_compaction_chunk_size_bytes_count 4.861956e+06 +prometheus_tsdb_compaction_duration_seconds_bucket{le="1"} 61 +prometheus_tsdb_compaction_duration_seconds_bucket{le="2"} 155 +prometheus_tsdb_compaction_duration_seconds_bucket{le="4"} 180 +prometheus_tsdb_compaction_duration_seconds_bucket{le="8"} 183 +prometheus_tsdb_compaction_duration_seconds_bucket{le="16"} 183 +prometheus_tsdb_compaction_duration_seconds_bucket{le="32"} 183 +prometheus_tsdb_compaction_duration_seconds_bucket{le="64"} 183 +prometheus_tsdb_compaction_duration_seconds_bucket{le="128"} 183 +prometheus_tsdb_compaction_duration_seconds_bucket{le="256"} 183 +prometheus_tsdb_compaction_duration_seconds_bucket{le="512"} 183 +prometheus_tsdb_compaction_duration_seconds_bucket{le="1024"} 183 +prometheus_tsdb_compaction_duration_seconds_bucket{le="2048"} 183 +prometheus_tsdb_compaction_duration_seconds_bucket{le="4096"} 183 +prometheus_tsdb_compaction_duration_seconds_bucket{le="8192"} 183 +prometheus_tsdb_compaction_duration_seconds_bucket{le="+Inf"} 183 +prometheus_tsdb_compaction_duration_seconds_sum 254.9095696899999 +prometheus_tsdb_compaction_duration_seconds_count 183 +prometheus_tsdb_compaction_populating_block 0 +prometheus_tsdb_compactions_failed_total 0 +prometheus_tsdb_compactions_skipped_total 0 +prometheus_tsdb_compactions_total 183 +prometheus_tsdb_compactions_triggered_total 14855 +prometheus_tsdb_data_replay_duration_seconds 5.550163617 +prometheus_tsdb_exemplar_exemplars_appended_total 0 +prometheus_tsdb_exemplar_exemplars_in_storage 0 +prometheus_tsdb_exemplar_last_exemplars_timestamp_seconds 0 +prometheus_tsdb_exemplar_max_exemplars 0 +prometheus_tsdb_exemplar_out_of_order_exemplars_total 0 +prometheus_tsdb_exemplar_series_with_exemplars_in_storage 0 +prometheus_tsdb_head_active_appenders 0 +prometheus_tsdb_head_chunks 31476 +prometheus_tsdb_head_chunks_created_total 4.893432e+06 +prometheus_tsdb_head_chunks_removed_total 4.861956e+06 +prometheus_tsdb_head_chunks_storage_size_bytes 7.237299e+06 +prometheus_tsdb_head_gc_duration_seconds_sum 4.773801686000001 +prometheus_tsdb_head_gc_duration_seconds_count 123 +prometheus_tsdb_head_max_time 1.738949375191e+12 +prometheus_tsdb_head_max_time_seconds 1.738949375e+09 +prometheus_tsdb_head_min_time 1.738944000171e+12 +prometheus_tsdb_head_min_time_seconds 1.738944e+09 +prometheus_tsdb_head_out_of_order_samples_appended_total{type="float"} 0 +prometheus_tsdb_head_out_of_order_samples_appended_total{type="histogram"} 0 +prometheus_tsdb_head_samples_appended_total{type="float"} 5.85543187e+08 +prometheus_tsdb_head_samples_appended_total{type="histogram"} 0 +prometheus_tsdb_head_series 10720 +prometheus_tsdb_head_series_created_total 18541 +prometheus_tsdb_head_series_not_found_total 0 +prometheus_tsdb_head_series_removed_total 7821 +prometheus_tsdb_head_truncations_failed_total 0 +prometheus_tsdb_head_truncations_total 123 +prometheus_tsdb_isolation_high_watermark 7.852949e+06 +prometheus_tsdb_isolation_low_watermark 7.852949e+06 +prometheus_tsdb_lowest_timestamp 1.73618640004e+12 +prometheus_tsdb_lowest_timestamp_seconds 1.7361864e+09 +prometheus_tsdb_mmap_chunk_corruptions_total 0 +prometheus_tsdb_mmap_chunks_total 4.851264e+06 +prometheus_tsdb_out_of_bound_samples_total{type="float"} 0 +prometheus_tsdb_out_of_order_samples_total{type="float"} 517 +prometheus_tsdb_out_of_order_samples_total{type="histogram"} 0 +prometheus_tsdb_reloads_failures_total 0 +prometheus_tsdb_reloads_total 14822 +prometheus_tsdb_retention_limit_bytes 0 +prometheus_tsdb_retention_limit_seconds 2.6784e+06 +prometheus_tsdb_size_retentions_total 0 +prometheus_tsdb_snapshot_replay_error_total 0 +prometheus_tsdb_storage_blocks_bytes 2.762863592e+09 +prometheus_tsdb_symbol_table_size_bytes 10616 +prometheus_tsdb_time_retentions_total 5 +prometheus_tsdb_tombstone_cleanup_seconds_bucket{le="+Inf"} 0 +prometheus_tsdb_tombstone_cleanup_seconds_sum 0 +prometheus_tsdb_tombstone_cleanup_seconds_count 0 +prometheus_tsdb_too_old_samples_total{type="float"} 0 +prometheus_tsdb_vertical_compactions_total 0 +prometheus_tsdb_wal_completed_pages_total 109271 +prometheus_tsdb_wal_corruptions_total 0 +prometheus_tsdb_wal_fsync_duration_seconds{quantile="0.5"} NaN +prometheus_tsdb_wal_fsync_duration_seconds{quantile="0.9"} NaN +prometheus_tsdb_wal_fsync_duration_seconds{quantile="0.99"} NaN +prometheus_tsdb_wal_fsync_duration_seconds_sum 4.842524568000002 +prometheus_tsdb_wal_fsync_duration_seconds_count 123 +prometheus_tsdb_wal_page_flushes_total 2.293951e+06 +prometheus_tsdb_wal_segment_current 24726 +prometheus_tsdb_wal_storage_size_bytes 6.9168385e+07 +prometheus_tsdb_wal_truncate_duration_seconds_sum 121.61954577099996 +prometheus_tsdb_wal_truncate_duration_seconds_count 62 +prometheus_tsdb_wal_truncations_failed_total 0 +prometheus_tsdb_wal_truncations_total 62 +prometheus_tsdb_wal_writes_failed_total 0 +prometheus_web_federation_errors_total 0 +prometheus_web_federation_warnings_total 0 +promhttp_metric_handler_requests_in_flight 1 +promhttp_metric_handler_requests_total{code="200"} 4.059092e+06 +promhttp_metric_handler_requests_total{code="500"} 0 +promhttp_metric_handler_requests_total{code="503"} 0 +# EOF diff --git a/model/textparse/testdata/alltypes.237mfs.prom.txt b/model/textparse/testdata/alltypes.237mfs.prom.txt new file mode 100644 index 0000000000..01138055cf --- /dev/null +++ b/model/textparse/testdata/alltypes.237mfs.prom.txt @@ -0,0 +1,2332 @@ +# HELP go_gc_cycles_automatic_gc_cycles_total Count of completed GC cycles generated by the Go runtime. Sourced from /gc/cycles/automatic:gc-cycles +# TYPE go_gc_cycles_automatic_gc_cycles_total counter +go_gc_cycles_automatic_gc_cycles_total 190932 +# HELP go_gc_cycles_forced_gc_cycles_total Count of completed GC cycles forced by the application. Sourced from /gc/cycles/forced:gc-cycles +# TYPE go_gc_cycles_forced_gc_cycles_total counter +go_gc_cycles_forced_gc_cycles_total 0 +# HELP go_gc_cycles_total_gc_cycles_total Count of all completed GC cycles. Sourced from /gc/cycles/total:gc-cycles +# TYPE go_gc_cycles_total_gc_cycles_total counter +go_gc_cycles_total_gc_cycles_total 190932 +# HELP go_gc_duration_seconds A summary of the wall-time pause (stop-the-world) duration in garbage collection cycles. +# TYPE go_gc_duration_seconds summary +go_gc_duration_seconds{quantile="0"} 7.6238e-05 +go_gc_duration_seconds{quantile="0.25"} 0.00010188 +go_gc_duration_seconds{quantile="0.5"} 0.000135819 +go_gc_duration_seconds{quantile="0.75"} 0.000156061 +go_gc_duration_seconds{quantile="1"} 0.002389378 +go_gc_duration_seconds_sum 44.985611511 +go_gc_duration_seconds_count 190932 +# HELP go_gc_gogc_percent Heap size target percentage configured by the user, otherwise 100. This value is set by the GOGC environment variable, and the runtime/debug.SetGCPercent function. Sourced from /gc/gogc:percent +# TYPE go_gc_gogc_percent gauge +go_gc_gogc_percent 75 +# HELP go_gc_gomemlimit_bytes Go runtime memory limit configured by the user, otherwise math.MaxInt64. This value is set by the GOMEMLIMIT environment variable, and the runtime/debug.SetMemoryLimit function. Sourced from /gc/gomemlimit:bytes +# TYPE go_gc_gomemlimit_bytes gauge +go_gc_gomemlimit_bytes 9.03676723e+08 +# HELP go_gc_heap_allocs_by_size_bytes Distribution of heap allocations by approximate size. Bucket counts increase monotonically. Note that this does not include tiny objects as defined by /gc/heap/tiny/allocs:objects, only tiny blocks. Sourced from /gc/heap/allocs-by-size:bytes +# TYPE go_gc_heap_allocs_by_size_bytes histogram +go_gc_heap_allocs_by_size_bytes_bucket{le="8.999999999999998"} 2.279966416e+09 +go_gc_heap_allocs_by_size_bytes_bucket{le="24.999999999999996"} 1.9429106442e+10 +go_gc_heap_allocs_by_size_bytes_bucket{le="64.99999999999999"} 3.2158220609e+10 +go_gc_heap_allocs_by_size_bytes_bucket{le="144.99999999999997"} 4.2309744198e+10 +go_gc_heap_allocs_by_size_bytes_bucket{le="320.99999999999994"} 4.3418919481e+10 +go_gc_heap_allocs_by_size_bytes_bucket{le="704.9999999999999"} 4.374631622e+10 +go_gc_heap_allocs_by_size_bytes_bucket{le="1536.9999999999998"} 4.3917578245e+10 +go_gc_heap_allocs_by_size_bytes_bucket{le="3200.9999999999995"} 4.396605609e+10 +go_gc_heap_allocs_by_size_bytes_bucket{le="6528.999999999999"} 4.4007501305e+10 +go_gc_heap_allocs_by_size_bytes_bucket{le="13568.999999999998"} 4.4020325917e+10 +go_gc_heap_allocs_by_size_bytes_bucket{le="27264.999999999996"} 4.4036187548e+10 +go_gc_heap_allocs_by_size_bytes_bucket{le="+Inf"} 4.4046288803e+10 +go_gc_heap_allocs_by_size_bytes_sum 5.937322571024e+12 +go_gc_heap_allocs_by_size_bytes_count 4.4046288803e+10 +# HELP go_gc_heap_allocs_bytes_total Cumulative sum of memory allocated to the heap by the application. Sourced from /gc/heap/allocs:bytes +# TYPE go_gc_heap_allocs_bytes_total counter +go_gc_heap_allocs_bytes_total 5.937322571024e+12 +# HELP go_gc_heap_allocs_objects_total Cumulative count of heap allocations triggered by the application. Note that this does not include tiny objects as defined by /gc/heap/tiny/allocs:objects, only tiny blocks. Sourced from /gc/heap/allocs:objects +# TYPE go_gc_heap_allocs_objects_total counter +go_gc_heap_allocs_objects_total 4.4046288803e+10 +# HELP go_gc_heap_frees_by_size_bytes Distribution of freed heap allocations by approximate size. Bucket counts increase monotonically. Note that this does not include tiny objects as defined by /gc/heap/tiny/allocs:objects, only tiny blocks. Sourced from /gc/heap/frees-by-size:bytes +# TYPE go_gc_heap_frees_by_size_bytes histogram +go_gc_heap_frees_by_size_bytes_bucket{le="8.999999999999998"} 2.279951045e+09 +go_gc_heap_frees_by_size_bytes_bucket{le="24.999999999999996"} 1.9428965693e+10 +go_gc_heap_frees_by_size_bytes_bucket{le="64.99999999999999"} 3.2157849717e+10 +go_gc_heap_frees_by_size_bytes_bucket{le="144.99999999999997"} 4.2309204178e+10 +go_gc_heap_frees_by_size_bytes_bucket{le="320.99999999999994"} 4.3418348856e+10 +go_gc_heap_frees_by_size_bytes_bucket{le="704.9999999999999"} 4.3745739652e+10 +go_gc_heap_frees_by_size_bytes_bucket{le="1536.9999999999998"} 4.3916999773e+10 +go_gc_heap_frees_by_size_bytes_bucket{le="3200.9999999999995"} 4.3965477112e+10 +go_gc_heap_frees_by_size_bytes_bucket{le="6528.999999999999"} 4.4006921621e+10 +go_gc_heap_frees_by_size_bytes_bucket{le="13568.999999999998"} 4.4019746017e+10 +go_gc_heap_frees_by_size_bytes_bucket{le="27264.999999999996"} 4.4035607466e+10 +go_gc_heap_frees_by_size_bytes_bucket{le="+Inf"} 4.4045708586e+10 +go_gc_heap_frees_by_size_bytes_sum 5.937239047736e+12 +go_gc_heap_frees_by_size_bytes_count 4.4045708586e+10 +# HELP go_gc_heap_frees_bytes_total Cumulative sum of heap memory freed by the garbage collector. Sourced from /gc/heap/frees:bytes +# TYPE go_gc_heap_frees_bytes_total counter +go_gc_heap_frees_bytes_total 5.937239047736e+12 +# HELP go_gc_heap_frees_objects_total Cumulative count of heap allocations whose storage was freed by the garbage collector. Note that this does not include tiny objects as defined by /gc/heap/tiny/allocs:objects, only tiny blocks. Sourced from /gc/heap/frees:objects +# TYPE go_gc_heap_frees_objects_total counter +go_gc_heap_frees_objects_total 4.4045708586e+10 +# HELP go_gc_heap_goal_bytes Heap size target for the end of the GC cycle. Sourced from /gc/heap/goal:bytes +# TYPE go_gc_heap_goal_bytes gauge +go_gc_heap_goal_bytes 1.0365948e+08 +# HELP go_gc_heap_live_bytes Heap memory occupied by live objects that were marked by the previous GC. Sourced from /gc/heap/live:bytes +# TYPE go_gc_heap_live_bytes gauge +go_gc_heap_live_bytes 5.8791024e+07 +# HELP go_gc_heap_objects_objects Number of objects, live or unswept, occupying heap memory. Sourced from /gc/heap/objects:objects +# TYPE go_gc_heap_objects_objects gauge +go_gc_heap_objects_objects 580217 +# HELP go_gc_heap_tiny_allocs_objects_total Count of small allocations that are packed together into blocks. These allocations are counted separately from other allocations because each individual allocation is not tracked by the runtime, only their block. Each block is already accounted for in allocs-by-size and frees-by-size. Sourced from /gc/heap/tiny/allocs:objects +# TYPE go_gc_heap_tiny_allocs_objects_total counter +go_gc_heap_tiny_allocs_objects_total 8.306064403e+09 +# HELP go_gc_limiter_last_enabled_gc_cycle GC cycle the last time the GC CPU limiter was enabled. This metric is useful for diagnosing the root cause of an out-of-memory error, because the limiter trades memory for CPU time when the GC's CPU time gets too high. This is most likely to occur with use of SetMemoryLimit. The first GC cycle is cycle 1, so a value of 0 indicates that it was never enabled. Sourced from /gc/limiter/last-enabled:gc-cycle +# TYPE go_gc_limiter_last_enabled_gc_cycle gauge +go_gc_limiter_last_enabled_gc_cycle 160896 +# HELP go_gc_pauses_seconds Deprecated. Prefer the identical /sched/pauses/total/gc:seconds. Sourced from /gc/pauses:seconds +# TYPE go_gc_pauses_seconds histogram +go_gc_pauses_seconds_bucket{le="6.399999999999999e-08"} 0 +go_gc_pauses_seconds_bucket{le="6.399999999999999e-07"} 0 +go_gc_pauses_seconds_bucket{le="7.167999999999999e-06"} 137212 +go_gc_pauses_seconds_bucket{le="8.191999999999999e-05"} 208425 +go_gc_pauses_seconds_bucket{le="0.0009175039999999999"} 376121 +go_gc_pauses_seconds_bucket{le="0.010485759999999998"} 381798 +go_gc_pauses_seconds_bucket{le="0.11744051199999998"} 381863 +go_gc_pauses_seconds_bucket{le="+Inf"} 381864 +go_gc_pauses_seconds_sum 20.343611904000003 +go_gc_pauses_seconds_count 381864 +# HELP go_gc_scan_globals_bytes The total amount of global variable space that is scannable. Sourced from /gc/scan/globals:bytes +# TYPE go_gc_scan_globals_bytes gauge +go_gc_scan_globals_bytes 555824 +# HELP go_gc_scan_heap_bytes The total amount of heap space that is scannable. Sourced from /gc/scan/heap:bytes +# TYPE go_gc_scan_heap_bytes gauge +go_gc_scan_heap_bytes 4.0986192e+07 +# HELP go_gc_scan_stack_bytes The number of bytes of stack that were scanned last GC cycle. Sourced from /gc/scan/stack:bytes +# TYPE go_gc_scan_stack_bytes gauge +go_gc_scan_stack_bytes 477760 +# HELP go_gc_scan_total_bytes The total amount space that is scannable. Sum of all metrics in /gc/scan. Sourced from /gc/scan/total:bytes +# TYPE go_gc_scan_total_bytes gauge +go_gc_scan_total_bytes 4.2019776e+07 +# HELP go_gc_stack_starting_size_bytes The stack size of new goroutines. Sourced from /gc/stack/starting-size:bytes +# TYPE go_gc_stack_starting_size_bytes gauge +go_gc_stack_starting_size_bytes 4096 +# HELP go_goroutines Number of goroutines that currently exist. +# TYPE go_goroutines gauge +go_goroutines 151 +# HELP go_info Information about the Go environment. +# TYPE go_info gauge +go_info{version="go1.23.4"} 1 +# HELP go_memstats_alloc_bytes Number of bytes allocated in heap and currently in use. Equals to /memory/classes/heap/objects:bytes. +# TYPE go_memstats_alloc_bytes gauge +go_memstats_alloc_bytes 8.3523288e+07 +# HELP go_memstats_alloc_bytes_total Total number of bytes allocated in heap until now, even if released already. Equals to /gc/heap/allocs:bytes. +# TYPE go_memstats_alloc_bytes_total counter +go_memstats_alloc_bytes_total 5.937322571024e+12 +# HELP go_memstats_buck_hash_sys_bytes Number of bytes used by the profiling bucket hash table. Equals to /memory/classes/profiling/buckets:bytes. +# TYPE go_memstats_buck_hash_sys_bytes gauge +go_memstats_buck_hash_sys_bytes 9.35076e+06 +# HELP go_memstats_frees_total Total number of heap objects frees. Equals to /gc/heap/frees:objects + /gc/heap/tiny/allocs:objects. +# TYPE go_memstats_frees_total counter +go_memstats_frees_total 5.2351772989e+10 +# HELP go_memstats_gc_sys_bytes Number of bytes used for garbage collection system metadata. Equals to /memory/classes/metadata/other:bytes. +# TYPE go_memstats_gc_sys_bytes gauge +go_memstats_gc_sys_bytes 5.450544e+06 +# HELP go_memstats_heap_alloc_bytes Number of heap bytes allocated and currently in use, same as go_memstats_alloc_bytes. Equals to /memory/classes/heap/objects:bytes. +# TYPE go_memstats_heap_alloc_bytes gauge +go_memstats_heap_alloc_bytes 8.3523288e+07 +# HELP go_memstats_heap_idle_bytes Number of heap bytes waiting to be used. Equals to /memory/classes/heap/released:bytes + /memory/classes/heap/free:bytes. +# TYPE go_memstats_heap_idle_bytes gauge +go_memstats_heap_idle_bytes 1.23691008e+08 +# HELP go_memstats_heap_inuse_bytes Number of heap bytes that are in use. Equals to /memory/classes/heap/objects:bytes + /memory/classes/heap/unused:bytes +# TYPE go_memstats_heap_inuse_bytes gauge +go_memstats_heap_inuse_bytes 9.56416e+07 +# HELP go_memstats_heap_objects Number of currently allocated objects. Equals to /gc/heap/objects:objects. +# TYPE go_memstats_heap_objects gauge +go_memstats_heap_objects 580217 +# HELP go_memstats_heap_released_bytes Number of heap bytes released to OS. Equals to /memory/classes/heap/released:bytes. +# TYPE go_memstats_heap_released_bytes gauge +go_memstats_heap_released_bytes 1.05897984e+08 +# HELP go_memstats_heap_sys_bytes Number of heap bytes obtained from system. Equals to /memory/classes/heap/objects:bytes + /memory/classes/heap/unused:bytes + /memory/classes/heap/released:bytes + /memory/classes/heap/free:bytes. +# TYPE go_memstats_heap_sys_bytes gauge +go_memstats_heap_sys_bytes 2.19332608e+08 +# HELP go_memstats_last_gc_time_seconds Number of seconds since 1970 of last garbage collection. +# TYPE go_memstats_last_gc_time_seconds gauge +go_memstats_last_gc_time_seconds 1.738949372347471e+09 +# HELP go_memstats_mallocs_total Total number of heap objects allocated, both live and gc-ed. Semantically a counter version for go_memstats_heap_objects gauge. Equals to /gc/heap/allocs:objects + /gc/heap/tiny/allocs:objects. +# TYPE go_memstats_mallocs_total counter +go_memstats_mallocs_total 5.2352353206e+10 +# HELP go_memstats_mcache_inuse_bytes Number of bytes in use by mcache structures. Equals to /memory/classes/metadata/mcache/inuse:bytes. +# TYPE go_memstats_mcache_inuse_bytes gauge +go_memstats_mcache_inuse_bytes 1200 +# HELP go_memstats_mcache_sys_bytes Number of bytes used for mcache structures obtained from system. Equals to /memory/classes/metadata/mcache/inuse:bytes + /memory/classes/metadata/mcache/free:bytes. +# TYPE go_memstats_mcache_sys_bytes gauge +go_memstats_mcache_sys_bytes 15600 +# HELP go_memstats_mspan_inuse_bytes Number of bytes in use by mspan structures. Equals to /memory/classes/metadata/mspan/inuse:bytes. +# TYPE go_memstats_mspan_inuse_bytes gauge +go_memstats_mspan_inuse_bytes 1.21232e+06 +# HELP go_memstats_mspan_sys_bytes Number of bytes used for mspan structures obtained from system. Equals to /memory/classes/metadata/mspan/inuse:bytes + /memory/classes/metadata/mspan/free:bytes. +# TYPE go_memstats_mspan_sys_bytes gauge +go_memstats_mspan_sys_bytes 2.13792e+06 +# HELP go_memstats_next_gc_bytes Number of heap bytes when next garbage collection will take place. Equals to /gc/heap/goal:bytes. +# TYPE go_memstats_next_gc_bytes gauge +go_memstats_next_gc_bytes 1.0365948e+08 +# HELP go_memstats_other_sys_bytes Number of bytes used for other system allocations. Equals to /memory/classes/other:bytes. +# TYPE go_memstats_other_sys_bytes gauge +go_memstats_other_sys_bytes 803696 +# HELP go_memstats_stack_inuse_bytes Number of bytes obtained from system for stack allocator in non-CGO environments. Equals to /memory/classes/heap/stacks:bytes. +# TYPE go_memstats_stack_inuse_bytes gauge +go_memstats_stack_inuse_bytes 2.818048e+06 +# HELP go_memstats_stack_sys_bytes Number of bytes obtained from system for stack allocator. Equals to /memory/classes/heap/stacks:bytes + /memory/classes/os-stacks:bytes. +# TYPE go_memstats_stack_sys_bytes gauge +go_memstats_stack_sys_bytes 2.818048e+06 +# HELP go_memstats_sys_bytes Number of bytes obtained from system. Equals to /memory/classes/total:byte. +# TYPE go_memstats_sys_bytes gauge +go_memstats_sys_bytes 2.39909176e+08 +# HELP go_sched_gomaxprocs_threads The current runtime.GOMAXPROCS setting, or the number of operating system threads that can execute user-level Go code simultaneously. Sourced from /sched/gomaxprocs:threads +# TYPE go_sched_gomaxprocs_threads gauge +go_sched_gomaxprocs_threads 1 +# HELP go_sched_goroutines_goroutines Count of live goroutines. Sourced from /sched/goroutines:goroutines +# TYPE go_sched_goroutines_goroutines gauge +go_sched_goroutines_goroutines 151 +# HELP go_sched_latencies_seconds Distribution of the time goroutines have spent in the scheduler in a runnable state before actually running. Bucket counts increase monotonically. Sourced from /sched/latencies:seconds +# TYPE go_sched_latencies_seconds histogram +go_sched_latencies_seconds_bucket{le="6.399999999999999e-08"} 5.0101035e+07 +go_sched_latencies_seconds_bucket{le="6.399999999999999e-07"} 7.4291762e+07 +go_sched_latencies_seconds_bucket{le="7.167999999999999e-06"} 8.1386236e+07 +go_sched_latencies_seconds_bucket{le="8.191999999999999e-05"} 8.2116161e+07 +go_sched_latencies_seconds_bucket{le="0.0009175039999999999"} 8.2802009e+07 +go_sched_latencies_seconds_bucket{le="0.010485759999999998"} 8.6274195e+07 +go_sched_latencies_seconds_bucket{le="0.11744051199999998"} 8.6724074e+07 +go_sched_latencies_seconds_bucket{le="+Inf"} 8.6726596e+07 +go_sched_latencies_seconds_sum 8266.758178496 +go_sched_latencies_seconds_count 8.6726596e+07 +# HELP go_sched_pauses_stopping_gc_seconds Distribution of individual GC-related stop-the-world stopping latencies. This is the time it takes from deciding to stop the world until all Ps are stopped. This is a subset of the total GC-related stop-the-world time (/sched/pauses/total/gc:seconds). During this time, some threads may be executing. Bucket counts increase monotonically. Sourced from /sched/pauses/stopping/gc:seconds +# TYPE go_sched_pauses_stopping_gc_seconds histogram +go_sched_pauses_stopping_gc_seconds_bucket{le="6.399999999999999e-08"} 0 +go_sched_pauses_stopping_gc_seconds_bucket{le="6.399999999999999e-07"} 236272 +go_sched_pauses_stopping_gc_seconds_bucket{le="7.167999999999999e-06"} 380799 +go_sched_pauses_stopping_gc_seconds_bucket{le="8.191999999999999e-05"} 381741 +go_sched_pauses_stopping_gc_seconds_bucket{le="0.0009175039999999999"} 381807 +go_sched_pauses_stopping_gc_seconds_bucket{le="0.010485759999999998"} 381862 +go_sched_pauses_stopping_gc_seconds_bucket{le="0.11744051199999998"} 381864 +go_sched_pauses_stopping_gc_seconds_bucket{le="+Inf"} 381864 +go_sched_pauses_stopping_gc_seconds_sum 0.191211904 +go_sched_pauses_stopping_gc_seconds_count 381864 +# HELP go_sched_pauses_stopping_other_seconds Distribution of individual non-GC-related stop-the-world stopping latencies. This is the time it takes from deciding to stop the world until all Ps are stopped. This is a subset of the total non-GC-related stop-the-world time (/sched/pauses/total/other:seconds). During this time, some threads may be executing. Bucket counts increase monotonically. Sourced from /sched/pauses/stopping/other:seconds +# TYPE go_sched_pauses_stopping_other_seconds histogram +go_sched_pauses_stopping_other_seconds_bucket{le="6.399999999999999e-08"} 0 +go_sched_pauses_stopping_other_seconds_bucket{le="6.399999999999999e-07"} 0 +go_sched_pauses_stopping_other_seconds_bucket{le="7.167999999999999e-06"} 0 +go_sched_pauses_stopping_other_seconds_bucket{le="8.191999999999999e-05"} 0 +go_sched_pauses_stopping_other_seconds_bucket{le="0.0009175039999999999"} 0 +go_sched_pauses_stopping_other_seconds_bucket{le="0.010485759999999998"} 0 +go_sched_pauses_stopping_other_seconds_bucket{le="0.11744051199999998"} 0 +go_sched_pauses_stopping_other_seconds_bucket{le="+Inf"} 0 +go_sched_pauses_stopping_other_seconds_sum 0 +go_sched_pauses_stopping_other_seconds_count 0 +# HELP go_sched_pauses_total_gc_seconds Distribution of individual GC-related stop-the-world pause latencies. This is the time from deciding to stop the world until the world is started again. Some of this time is spent getting all threads to stop (this is measured directly in /sched/pauses/stopping/gc:seconds), during which some threads may still be running. Bucket counts increase monotonically. Sourced from /sched/pauses/total/gc:seconds +# TYPE go_sched_pauses_total_gc_seconds histogram +go_sched_pauses_total_gc_seconds_bucket{le="6.399999999999999e-08"} 0 +go_sched_pauses_total_gc_seconds_bucket{le="6.399999999999999e-07"} 0 +go_sched_pauses_total_gc_seconds_bucket{le="7.167999999999999e-06"} 137212 +go_sched_pauses_total_gc_seconds_bucket{le="8.191999999999999e-05"} 208425 +go_sched_pauses_total_gc_seconds_bucket{le="0.0009175039999999999"} 376121 +go_sched_pauses_total_gc_seconds_bucket{le="0.010485759999999998"} 381798 +go_sched_pauses_total_gc_seconds_bucket{le="0.11744051199999998"} 381863 +go_sched_pauses_total_gc_seconds_bucket{le="+Inf"} 381864 +go_sched_pauses_total_gc_seconds_sum 20.343611904000003 +go_sched_pauses_total_gc_seconds_count 381864 +# HELP go_sched_pauses_total_other_seconds Distribution of individual non-GC-related stop-the-world pause latencies. This is the time from deciding to stop the world until the world is started again. Some of this time is spent getting all threads to stop (measured directly in /sched/pauses/stopping/other:seconds). Bucket counts increase monotonically. Sourced from /sched/pauses/total/other:seconds +# TYPE go_sched_pauses_total_other_seconds histogram +go_sched_pauses_total_other_seconds_bucket{le="6.399999999999999e-08"} 0 +go_sched_pauses_total_other_seconds_bucket{le="6.399999999999999e-07"} 0 +go_sched_pauses_total_other_seconds_bucket{le="7.167999999999999e-06"} 0 +go_sched_pauses_total_other_seconds_bucket{le="8.191999999999999e-05"} 0 +go_sched_pauses_total_other_seconds_bucket{le="0.0009175039999999999"} 0 +go_sched_pauses_total_other_seconds_bucket{le="0.010485759999999998"} 0 +go_sched_pauses_total_other_seconds_bucket{le="0.11744051199999998"} 0 +go_sched_pauses_total_other_seconds_bucket{le="+Inf"} 0 +go_sched_pauses_total_other_seconds_sum 0 +go_sched_pauses_total_other_seconds_count 0 +# HELP go_sync_mutex_wait_total_seconds_total Approximate cumulative time goroutines have spent blocked on a sync.Mutex, sync.RWMutex, or runtime-internal lock. This metric is useful for identifying global changes in lock contention. Collect a mutex or block profile using the runtime/pprof package for more detailed contention data. Sourced from /sync/mutex/wait/total:seconds +# TYPE go_sync_mutex_wait_total_seconds_total counter +go_sync_mutex_wait_total_seconds_total 628.29966272 +# HELP go_threads Number of OS threads created. +# TYPE go_threads gauge +go_threads 10 +# HELP net_conntrack_dialer_conn_attempted_total Total number of connections attempted by the given dialer a given name. +# TYPE net_conntrack_dialer_conn_attempted_total counter +net_conntrack_dialer_conn_attempted_total{dialer_name="alertmanager"} 2 +net_conntrack_dialer_conn_attempted_total{dialer_name="blackbox"} 1 +net_conntrack_dialer_conn_attempted_total{dialer_name="caddy"} 1 +net_conntrack_dialer_conn_attempted_total{dialer_name="cadvisor"} 2 +net_conntrack_dialer_conn_attempted_total{dialer_name="default"} 0 +net_conntrack_dialer_conn_attempted_total{dialer_name="grafana"} 7 +net_conntrack_dialer_conn_attempted_total{dialer_name="node"} 5 +net_conntrack_dialer_conn_attempted_total{dialer_name="prometheus"} 1 +net_conntrack_dialer_conn_attempted_total{dialer_name="random"} 4 +# HELP net_conntrack_dialer_conn_closed_total Total number of connections closed which originated from the dialer of a given name. +# TYPE net_conntrack_dialer_conn_closed_total counter +net_conntrack_dialer_conn_closed_total{dialer_name="alertmanager"} 0 +net_conntrack_dialer_conn_closed_total{dialer_name="blackbox"} 0 +net_conntrack_dialer_conn_closed_total{dialer_name="caddy"} 0 +net_conntrack_dialer_conn_closed_total{dialer_name="cadvisor"} 1 +net_conntrack_dialer_conn_closed_total{dialer_name="default"} 0 +net_conntrack_dialer_conn_closed_total{dialer_name="grafana"} 4 +net_conntrack_dialer_conn_closed_total{dialer_name="node"} 4 +net_conntrack_dialer_conn_closed_total{dialer_name="prometheus"} 0 +net_conntrack_dialer_conn_closed_total{dialer_name="random"} 0 +# HELP net_conntrack_dialer_conn_established_total Total number of connections successfully established by the given dialer a given name. +# TYPE net_conntrack_dialer_conn_established_total counter +net_conntrack_dialer_conn_established_total{dialer_name="alertmanager"} 2 +net_conntrack_dialer_conn_established_total{dialer_name="blackbox"} 1 +net_conntrack_dialer_conn_established_total{dialer_name="caddy"} 1 +net_conntrack_dialer_conn_established_total{dialer_name="cadvisor"} 2 +net_conntrack_dialer_conn_established_total{dialer_name="default"} 0 +net_conntrack_dialer_conn_established_total{dialer_name="grafana"} 5 +net_conntrack_dialer_conn_established_total{dialer_name="node"} 5 +net_conntrack_dialer_conn_established_total{dialer_name="prometheus"} 1 +net_conntrack_dialer_conn_established_total{dialer_name="random"} 4 +# HELP net_conntrack_dialer_conn_failed_total Total number of connections failed to dial by the dialer a given name. +# TYPE net_conntrack_dialer_conn_failed_total counter +net_conntrack_dialer_conn_failed_total{dialer_name="alertmanager",reason="refused"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="alertmanager",reason="resolution"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="alertmanager",reason="timeout"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="alertmanager",reason="unknown"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="blackbox",reason="refused"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="blackbox",reason="resolution"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="blackbox",reason="timeout"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="blackbox",reason="unknown"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="caddy",reason="refused"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="caddy",reason="resolution"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="caddy",reason="timeout"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="caddy",reason="unknown"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="cadvisor",reason="refused"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="cadvisor",reason="resolution"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="cadvisor",reason="timeout"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="cadvisor",reason="unknown"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="default",reason="refused"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="default",reason="resolution"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="default",reason="timeout"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="default",reason="unknown"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="grafana",reason="refused"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="grafana",reason="resolution"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="grafana",reason="timeout"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="grafana",reason="unknown"} 2 +net_conntrack_dialer_conn_failed_total{dialer_name="node",reason="refused"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="node",reason="resolution"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="node",reason="timeout"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="node",reason="unknown"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="prometheus",reason="refused"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="prometheus",reason="resolution"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="prometheus",reason="timeout"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="prometheus",reason="unknown"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="random",reason="refused"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="random",reason="resolution"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="random",reason="timeout"} 0 +net_conntrack_dialer_conn_failed_total{dialer_name="random",reason="unknown"} 0 +# HELP net_conntrack_listener_conn_accepted_total Total number of connections opened to the listener of a given name. +# TYPE net_conntrack_listener_conn_accepted_total counter +net_conntrack_listener_conn_accepted_total{listener_name="http"} 561771 +# HELP net_conntrack_listener_conn_closed_total Total number of connections closed that were made to the listener of a given name. +# TYPE net_conntrack_listener_conn_closed_total counter +net_conntrack_listener_conn_closed_total{listener_name="http"} 561723 +# HELP process_cpu_seconds_total Total user and system CPU time spent in seconds. +# TYPE process_cpu_seconds_total counter +process_cpu_seconds_total 96501.42 +# HELP process_max_fds Maximum number of open file descriptors. +# TYPE process_max_fds gauge +process_max_fds 65000 +# HELP process_network_receive_bytes_total Number of bytes received by the process over the network. +# TYPE process_network_receive_bytes_total counter +process_network_receive_bytes_total 9.89686846606e+11 +# HELP process_network_transmit_bytes_total Number of bytes sent by the process over the network. +# TYPE process_network_transmit_bytes_total counter +process_network_transmit_bytes_total 3.743928187168e+12 +# HELP process_open_fds Number of open file descriptors. +# TYPE process_open_fds gauge +process_open_fds 124 +# HELP process_resident_memory_bytes Resident memory size in bytes. +# TYPE process_resident_memory_bytes gauge +process_resident_memory_bytes 1.72822528e+08 +# HELP process_start_time_seconds Start time of the process since unix epoch in seconds. +# TYPE process_start_time_seconds gauge +process_start_time_seconds 1.73806369023e+09 +# HELP process_virtual_memory_bytes Virtual memory size in bytes. +# TYPE process_virtual_memory_bytes gauge +process_virtual_memory_bytes 4.608667648e+09 +# HELP process_virtual_memory_max_bytes Maximum amount of virtual memory available in bytes. +# TYPE process_virtual_memory_max_bytes gauge +process_virtual_memory_max_bytes 1.8446744073709552e+19 +# HELP prometheus_api_notification_active_subscribers The current number of active notification subscribers. +# TYPE prometheus_api_notification_active_subscribers gauge +prometheus_api_notification_active_subscribers 2 +# HELP prometheus_api_notification_updates_dropped_total Total number of notification updates dropped. +# TYPE prometheus_api_notification_updates_dropped_total counter +prometheus_api_notification_updates_dropped_total 0 +# HELP prometheus_api_notification_updates_sent_total Total number of notification updates sent. +# TYPE prometheus_api_notification_updates_sent_total counter +prometheus_api_notification_updates_sent_total 5 +# HELP prometheus_api_remote_read_queries The current number of remote read queries being executed or waiting. +# TYPE prometheus_api_remote_read_queries gauge +prometheus_api_remote_read_queries 0 +# HELP prometheus_build_info A metric with a constant '1' value labeled by version, revision, branch, goversion from which prometheus was built, and the goos and goarch for the build. +# TYPE prometheus_build_info gauge +prometheus_build_info{branch="HEAD",goarch="amd64",goos="linux",goversion="go1.23.4",revision="7086161a93b262aa0949dbf2aba15a5a7b13e0a3",tags="netgo,builtinassets,stringlabels",version="3.1.0"} 1 +# HELP prometheus_config_last_reload_success_timestamp_seconds Timestamp of the last successful configuration reload. +# TYPE prometheus_config_last_reload_success_timestamp_seconds gauge +prometheus_config_last_reload_success_timestamp_seconds 1.7380636976181264e+09 +# HELP prometheus_config_last_reload_successful Whether the last configuration reload attempt was successful. +# TYPE prometheus_config_last_reload_successful gauge +prometheus_config_last_reload_successful 1 +# HELP prometheus_engine_queries The current number of queries being executed or waiting. +# TYPE prometheus_engine_queries gauge +prometheus_engine_queries 0 +# HELP prometheus_engine_queries_concurrent_max The max number of concurrent queries. +# TYPE prometheus_engine_queries_concurrent_max gauge +prometheus_engine_queries_concurrent_max 20 +# HELP prometheus_engine_query_duration_seconds Query timings +# TYPE prometheus_engine_query_duration_seconds summary +prometheus_engine_query_duration_seconds{slice="inner_eval",quantile="0.5"} 8.075e-05 +prometheus_engine_query_duration_seconds{slice="inner_eval",quantile="0.9"} 0.000917449 +prometheus_engine_query_duration_seconds{slice="inner_eval",quantile="0.99"} 0.009315769 +prometheus_engine_query_duration_seconds_sum{slice="inner_eval"} 12506.67007997419 +prometheus_engine_query_duration_seconds_count{slice="inner_eval"} 8.714071e+06 +prometheus_engine_query_duration_seconds{slice="prepare_time",quantile="0.5"} 2.228e-05 +prometheus_engine_query_duration_seconds{slice="prepare_time",quantile="0.9"} 6.2819e-05 +prometheus_engine_query_duration_seconds{slice="prepare_time",quantile="0.99"} 0.000399637 +prometheus_engine_query_duration_seconds_sum{slice="prepare_time"} 490.326247638047 +prometheus_engine_query_duration_seconds_count{slice="prepare_time"} 8.714071e+06 +prometheus_engine_query_duration_seconds{slice="queue_time",quantile="0.5"} 4.628e-06 +prometheus_engine_query_duration_seconds{slice="queue_time",quantile="0.9"} 1.6082e-05 +prometheus_engine_query_duration_seconds{slice="queue_time",quantile="0.99"} 4.1174e-05 +prometheus_engine_query_duration_seconds_sum{slice="queue_time"} 8720.662071224393 +prometheus_engine_query_duration_seconds_count{slice="queue_time"} 1.7428526e+07 +prometheus_engine_query_duration_seconds{slice="result_sort",quantile="0.5"} 7.83e-07 +prometheus_engine_query_duration_seconds{slice="result_sort",quantile="0.9"} 1.994e-06 +prometheus_engine_query_duration_seconds{slice="result_sort",quantile="0.99"} 1.458e-05 +prometheus_engine_query_duration_seconds_sum{slice="result_sort"} 2.337879348999977 +prometheus_engine_query_duration_seconds_count{slice="result_sort"} 1.208154e+06 +# HELP prometheus_engine_query_log_enabled State of the query log. +# TYPE prometheus_engine_query_log_enabled gauge +prometheus_engine_query_log_enabled 0 +# HELP prometheus_engine_query_log_failures_total The number of query log failures. +# TYPE prometheus_engine_query_log_failures_total counter +prometheus_engine_query_log_failures_total 0 +# HELP prometheus_engine_query_samples_total The total number of samples loaded by all queries. +# TYPE prometheus_engine_query_samples_total counter +prometheus_engine_query_samples_total 9.1183747239e+10 +# HELP prometheus_http_request_duration_seconds Histogram of latencies for HTTP requests. +# TYPE prometheus_http_request_duration_seconds histogram +prometheus_http_request_duration_seconds_bucket{handler="/",le="0.1"} 688 +prometheus_http_request_duration_seconds_bucket{handler="/",le="0.2"} 688 +prometheus_http_request_duration_seconds_bucket{handler="/",le="0.4"} 688 +prometheus_http_request_duration_seconds_bucket{handler="/",le="1"} 688 +prometheus_http_request_duration_seconds_bucket{handler="/",le="3"} 688 +prometheus_http_request_duration_seconds_bucket{handler="/",le="8"} 688 +prometheus_http_request_duration_seconds_bucket{handler="/",le="20"} 688 +prometheus_http_request_duration_seconds_bucket{handler="/",le="60"} 688 +prometheus_http_request_duration_seconds_bucket{handler="/",le="120"} 688 +prometheus_http_request_duration_seconds_bucket{handler="/",le="+Inf"} 688 +prometheus_http_request_duration_seconds_sum{handler="/"} 0.026826932000000022 +prometheus_http_request_duration_seconds_count{handler="/"} 688 +prometheus_http_request_duration_seconds_bucket{handler="/-/healthy",le="0.1"} 29524 +prometheus_http_request_duration_seconds_bucket{handler="/-/healthy",le="0.2"} 29524 +prometheus_http_request_duration_seconds_bucket{handler="/-/healthy",le="0.4"} 29524 +prometheus_http_request_duration_seconds_bucket{handler="/-/healthy",le="1"} 29524 +prometheus_http_request_duration_seconds_bucket{handler="/-/healthy",le="3"} 29524 +prometheus_http_request_duration_seconds_bucket{handler="/-/healthy",le="8"} 29524 +prometheus_http_request_duration_seconds_bucket{handler="/-/healthy",le="20"} 29524 +prometheus_http_request_duration_seconds_bucket{handler="/-/healthy",le="60"} 29524 +prometheus_http_request_duration_seconds_bucket{handler="/-/healthy",le="120"} 29524 +prometheus_http_request_duration_seconds_bucket{handler="/-/healthy",le="+Inf"} 29524 +prometheus_http_request_duration_seconds_sum{handler="/-/healthy"} 0.7400570460000002 +prometheus_http_request_duration_seconds_count{handler="/-/healthy"} 29524 +prometheus_http_request_duration_seconds_bucket{handler="/-/ready",le="0.1"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/-/ready",le="0.2"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/-/ready",le="0.4"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/-/ready",le="1"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/-/ready",le="3"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/-/ready",le="8"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/-/ready",le="20"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/-/ready",le="60"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/-/ready",le="120"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/-/ready",le="+Inf"} 49 +prometheus_http_request_duration_seconds_sum{handler="/-/ready"} 0.005040123000000002 +prometheus_http_request_duration_seconds_count{handler="/-/ready"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/alerts",le="0.1"} 48 +prometheus_http_request_duration_seconds_bucket{handler="/alerts",le="0.2"} 48 +prometheus_http_request_duration_seconds_bucket{handler="/alerts",le="0.4"} 48 +prometheus_http_request_duration_seconds_bucket{handler="/alerts",le="1"} 48 +prometheus_http_request_duration_seconds_bucket{handler="/alerts",le="3"} 48 +prometheus_http_request_duration_seconds_bucket{handler="/alerts",le="8"} 48 +prometheus_http_request_duration_seconds_bucket{handler="/alerts",le="20"} 48 +prometheus_http_request_duration_seconds_bucket{handler="/alerts",le="60"} 48 +prometheus_http_request_duration_seconds_bucket{handler="/alerts",le="120"} 48 +prometheus_http_request_duration_seconds_bucket{handler="/alerts",le="+Inf"} 48 +prometheus_http_request_duration_seconds_sum{handler="/alerts"} 0.011801602999999999 +prometheus_http_request_duration_seconds_count{handler="/alerts"} 48 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/*path",le="0.1"} 27 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/*path",le="0.2"} 27 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/*path",le="0.4"} 27 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/*path",le="1"} 27 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/*path",le="3"} 27 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/*path",le="8"} 27 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/*path",le="20"} 27 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/*path",le="60"} 27 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/*path",le="120"} 27 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/*path",le="+Inf"} 27 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/*path"} 0.001724389 +prometheus_http_request_duration_seconds_count{handler="/api/v1/*path"} 27 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/alertmanagers",le="0.1"} 8 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/alertmanagers",le="0.2"} 8 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/alertmanagers",le="0.4"} 8 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/alertmanagers",le="1"} 8 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/alertmanagers",le="3"} 8 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/alertmanagers",le="8"} 8 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/alertmanagers",le="20"} 8 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/alertmanagers",le="60"} 8 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/alertmanagers",le="120"} 8 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/alertmanagers",le="+Inf"} 8 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/alertmanagers"} 0.042492975999999995 +prometheus_http_request_duration_seconds_count{handler="/api/v1/alertmanagers"} 8 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/alerts",le="0.1"} 14630 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/alerts",le="0.2"} 14635 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/alerts",le="0.4"} 14635 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/alerts",le="1"} 14635 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/alerts",le="3"} 14635 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/alerts",le="8"} 14635 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/alerts",le="20"} 14635 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/alerts",le="60"} 14635 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/alerts",le="120"} 14635 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/alerts",le="+Inf"} 14635 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/alerts"} 19.028669391999912 +prometheus_http_request_duration_seconds_count{handler="/api/v1/alerts"} 14635 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/format_query",le="0.1"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/format_query",le="0.2"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/format_query",le="0.4"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/format_query",le="1"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/format_query",le="3"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/format_query",le="8"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/format_query",le="20"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/format_query",le="60"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/format_query",le="120"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/format_query",le="+Inf"} 4 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/format_query"} 0.023786675 +prometheus_http_request_duration_seconds_count{handler="/api/v1/format_query"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/label/:name/values",le="0.1"} 17773 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/label/:name/values",le="0.2"} 17860 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/label/:name/values",le="0.4"} 17939 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/label/:name/values",le="1"} 17970 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/label/:name/values",le="3"} 17971 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/label/:name/values",le="8"} 17971 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/label/:name/values",le="20"} 17971 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/label/:name/values",le="60"} 17971 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/label/:name/values",le="120"} 17971 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/label/:name/values",le="+Inf"} 17971 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/label/:name/values"} 99.95326355699925 +prometheus_http_request_duration_seconds_count{handler="/api/v1/label/:name/values"} 17971 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/labels",le="0.1"} 4905 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/labels",le="0.2"} 4912 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/labels",le="0.4"} 4916 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/labels",le="1"} 4921 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/labels",le="3"} 4921 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/labels",le="8"} 4921 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/labels",le="20"} 4921 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/labels",le="60"} 4921 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/labels",le="120"} 4921 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/labels",le="+Inf"} 4921 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/labels"} 12.963155722000025 +prometheus_http_request_duration_seconds_count{handler="/api/v1/labels"} 4921 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/metadata",le="0.1"} 2073 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/metadata",le="0.2"} 2091 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/metadata",le="0.4"} 2093 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/metadata",le="1"} 2093 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/metadata",le="3"} 2093 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/metadata",le="8"} 2093 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/metadata",le="20"} 2093 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/metadata",le="60"} 2093 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/metadata",le="120"} 2093 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/metadata",le="+Inf"} 2093 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/metadata"} 16.90794258600001 +prometheus_http_request_duration_seconds_count{handler="/api/v1/metadata"} 2093 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/notifications",le="0.1"} 12 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/notifications",le="0.2"} 12 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/notifications",le="0.4"} 12 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/notifications",le="1"} 12 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/notifications",le="3"} 12 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/notifications",le="8"} 12 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/notifications",le="20"} 12 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/notifications",le="60"} 12 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/notifications",le="120"} 12 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/notifications",le="+Inf"} 12 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/notifications"} 0.029363403 +prometheus_http_request_duration_seconds_count{handler="/api/v1/notifications"} 12 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/notifications/live",le="0.1"} 3 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/notifications/live",le="0.2"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/notifications/live",le="0.4"} 11 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/notifications/live",le="1"} 29 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/notifications/live",le="3"} 79 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/notifications/live",le="8"} 116 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/notifications/live",le="20"} 187 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/notifications/live",le="60"} 224 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/notifications/live",le="120"} 234 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/notifications/live",le="+Inf"} 263 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/notifications/live"} 85150.78366682903 +prometheus_http_request_duration_seconds_count{handler="/api/v1/notifications/live"} 263 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/parse_query",le="0.1"} 265 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/parse_query",le="0.2"} 265 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/parse_query",le="0.4"} 265 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/parse_query",le="1"} 265 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/parse_query",le="3"} 265 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/parse_query",le="8"} 265 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/parse_query",le="20"} 265 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/parse_query",le="60"} 265 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/parse_query",le="120"} 265 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/parse_query",le="+Inf"} 265 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/parse_query"} 0.7265021480000003 +prometheus_http_request_duration_seconds_count{handler="/api/v1/parse_query"} 265 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query",le="0.1"} 3051 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query",le="0.2"} 3082 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query",le="0.4"} 3173 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query",le="1"} 3213 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query",le="3"} 3216 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query",le="8"} 3217 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query",le="20"} 3217 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query",le="60"} 3217 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query",le="120"} 3217 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query",le="+Inf"} 3217 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/query"} 70.23571750499987 +prometheus_http_request_duration_seconds_count{handler="/api/v1/query"} 3217 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query_exemplars",le="0.1"} 831 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query_exemplars",le="0.2"} 839 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query_exemplars",le="0.4"} 841 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query_exemplars",le="1"} 841 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query_exemplars",le="3"} 841 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query_exemplars",le="8"} 841 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query_exemplars",le="20"} 841 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query_exemplars",le="60"} 841 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query_exemplars",le="120"} 841 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query_exemplars",le="+Inf"} 841 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/query_exemplars"} 3.6569132899999968 +prometheus_http_request_duration_seconds_count{handler="/api/v1/query_exemplars"} 841 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query_range",le="0.1"} 116338 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query_range",le="0.2"} 116784 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query_range",le="0.4"} 117924 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query_range",le="1"} 118147 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query_range",le="3"} 118155 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query_range",le="8"} 118157 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query_range",le="20"} 118157 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query_range",le="60"} 118157 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query_range",le="120"} 118157 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/query_range",le="+Inf"} 118157 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/query_range"} 832.1240439999854 +prometheus_http_request_duration_seconds_count{handler="/api/v1/query_range"} 118157 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/read",le="0.1"} 16 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/read",le="0.2"} 16 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/read",le="0.4"} 16 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/read",le="1"} 16 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/read",le="3"} 16 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/read",le="8"} 16 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/read",le="20"} 16 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/read",le="60"} 16 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/read",le="120"} 16 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/read",le="+Inf"} 16 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/read"} 0.07308876400000001 +prometheus_http_request_duration_seconds_count{handler="/api/v1/read"} 16 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/rules",le="0.1"} 88415 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/rules",le="0.2"} 88627 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/rules",le="0.4"} 88659 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/rules",le="1"} 88661 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/rules",le="3"} 88661 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/rules",le="8"} 88662 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/rules",le="20"} 88662 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/rules",le="60"} 88662 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/rules",le="120"} 88662 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/rules",le="+Inf"} 88662 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/rules"} 629.5237672670012 +prometheus_http_request_duration_seconds_count{handler="/api/v1/rules"} 88662 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/scrape_pools",le="0.1"} 142 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/scrape_pools",le="0.2"} 142 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/scrape_pools",le="0.4"} 142 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/scrape_pools",le="1"} 142 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/scrape_pools",le="3"} 142 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/scrape_pools",le="8"} 142 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/scrape_pools",le="20"} 142 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/scrape_pools",le="60"} 142 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/scrape_pools",le="120"} 142 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/scrape_pools",le="+Inf"} 142 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/scrape_pools"} 0.17501777799999996 +prometheus_http_request_duration_seconds_count{handler="/api/v1/scrape_pools"} 142 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/series",le="0.1"} 1217 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/series",le="0.2"} 1236 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/series",le="0.4"} 1244 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/series",le="1"} 1245 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/series",le="3"} 1245 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/series",le="8"} 1245 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/series",le="20"} 1246 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/series",le="60"} 1246 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/series",le="120"} 1246 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/series",le="+Inf"} 1246 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/series"} 19.03228310300001 +prometheus_http_request_duration_seconds_count{handler="/api/v1/series"} 1246 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/buildinfo",le="0.1"} 4412 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/buildinfo",le="0.2"} 4413 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/buildinfo",le="0.4"} 4413 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/buildinfo",le="1"} 4413 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/buildinfo",le="3"} 4413 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/buildinfo",le="8"} 4413 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/buildinfo",le="20"} 4413 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/buildinfo",le="60"} 4413 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/buildinfo",le="120"} 4413 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/buildinfo",le="+Inf"} 4413 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/status/buildinfo"} 5.222974727000008 +prometheus_http_request_duration_seconds_count{handler="/api/v1/status/buildinfo"} 4413 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/config",le="0.1"} 84669 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/config",le="0.2"} 84716 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/config",le="0.4"} 84721 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/config",le="1"} 84723 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/config",le="3"} 84723 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/config",le="8"} 84723 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/config",le="20"} 84723 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/config",le="60"} 84723 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/config",le="120"} 84723 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/config",le="+Inf"} 84723 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/status/config"} 228.93575751199964 +prometheus_http_request_duration_seconds_count{handler="/api/v1/status/config"} 84723 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/flags",le="0.1"} 32 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/flags",le="0.2"} 32 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/flags",le="0.4"} 32 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/flags",le="1"} 32 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/flags",le="3"} 32 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/flags",le="8"} 32 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/flags",le="20"} 32 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/flags",le="60"} 32 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/flags",le="120"} 32 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/flags",le="+Inf"} 32 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/status/flags"} 0.093302815 +prometheus_http_request_duration_seconds_count{handler="/api/v1/status/flags"} 32 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/runtimeinfo",le="0.1"} 862 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/runtimeinfo",le="0.2"} 863 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/runtimeinfo",le="0.4"} 863 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/runtimeinfo",le="1"} 863 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/runtimeinfo",le="3"} 863 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/runtimeinfo",le="8"} 863 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/runtimeinfo",le="20"} 863 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/runtimeinfo",le="60"} 863 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/runtimeinfo",le="120"} 863 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/runtimeinfo",le="+Inf"} 863 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/status/runtimeinfo"} 8.009312817000001 +prometheus_http_request_duration_seconds_count{handler="/api/v1/status/runtimeinfo"} 863 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/tsdb",le="0.1"} 94 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/tsdb",le="0.2"} 94 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/tsdb",le="0.4"} 94 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/tsdb",le="1"} 94 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/tsdb",le="3"} 94 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/tsdb",le="8"} 94 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/tsdb",le="20"} 94 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/tsdb",le="60"} 94 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/tsdb",le="120"} 94 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/tsdb",le="+Inf"} 94 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/status/tsdb"} 1.3378732679999994 +prometheus_http_request_duration_seconds_count{handler="/api/v1/status/tsdb"} 94 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/walreplay",le="0.1"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/walreplay",le="0.2"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/walreplay",le="0.4"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/walreplay",le="1"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/walreplay",le="3"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/walreplay",le="8"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/walreplay",le="20"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/walreplay",le="60"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/walreplay",le="120"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/status/walreplay",le="+Inf"} 49 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/status/walreplay"} 0.034643267 +prometheus_http_request_duration_seconds_count{handler="/api/v1/status/walreplay"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/targets",le="0.1"} 191 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/targets",le="0.2"} 191 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/targets",le="0.4"} 191 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/targets",le="1"} 191 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/targets",le="3"} 191 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/targets",le="8"} 191 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/targets",le="20"} 191 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/targets",le="60"} 191 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/targets",le="120"} 191 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/targets",le="+Inf"} 191 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/targets"} 0.5432347889999999 +prometheus_http_request_duration_seconds_count{handler="/api/v1/targets"} 191 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/targets/metadata",le="0.1"} 18 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/targets/metadata",le="0.2"} 18 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/targets/metadata",le="0.4"} 18 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/targets/metadata",le="1"} 18 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/targets/metadata",le="3"} 18 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/targets/metadata",le="8"} 18 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/targets/metadata",le="20"} 18 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/targets/metadata",le="60"} 18 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/targets/metadata",le="120"} 18 +prometheus_http_request_duration_seconds_bucket{handler="/api/v1/targets/metadata",le="+Inf"} 18 +prometheus_http_request_duration_seconds_sum{handler="/api/v1/targets/metadata"} 0.043789973 +prometheus_http_request_duration_seconds_count{handler="/api/v1/targets/metadata"} 18 +prometheus_http_request_duration_seconds_bucket{handler="/assets/*filepath",le="0.1"} 385 +prometheus_http_request_duration_seconds_bucket{handler="/assets/*filepath",le="0.2"} 481 +prometheus_http_request_duration_seconds_bucket{handler="/assets/*filepath",le="0.4"} 528 +prometheus_http_request_duration_seconds_bucket{handler="/assets/*filepath",le="1"} 612 +prometheus_http_request_duration_seconds_bucket{handler="/assets/*filepath",le="3"} 634 +prometheus_http_request_duration_seconds_bucket{handler="/assets/*filepath",le="8"} 640 +prometheus_http_request_duration_seconds_bucket{handler="/assets/*filepath",le="20"} 641 +prometheus_http_request_duration_seconds_bucket{handler="/assets/*filepath",le="60"} 642 +prometheus_http_request_duration_seconds_bucket{handler="/assets/*filepath",le="120"} 643 +prometheus_http_request_duration_seconds_bucket{handler="/assets/*filepath",le="+Inf"} 643 +prometheus_http_request_duration_seconds_sum{handler="/assets/*filepath"} 280.0660388799997 +prometheus_http_request_duration_seconds_count{handler="/assets/*filepath"} 643 +prometheus_http_request_duration_seconds_bucket{handler="/classic/static/*filepath",le="0.1"} 103 +prometheus_http_request_duration_seconds_bucket{handler="/classic/static/*filepath",le="0.2"} 103 +prometheus_http_request_duration_seconds_bucket{handler="/classic/static/*filepath",le="0.4"} 103 +prometheus_http_request_duration_seconds_bucket{handler="/classic/static/*filepath",le="1"} 103 +prometheus_http_request_duration_seconds_bucket{handler="/classic/static/*filepath",le="3"} 103 +prometheus_http_request_duration_seconds_bucket{handler="/classic/static/*filepath",le="8"} 103 +prometheus_http_request_duration_seconds_bucket{handler="/classic/static/*filepath",le="20"} 103 +prometheus_http_request_duration_seconds_bucket{handler="/classic/static/*filepath",le="60"} 103 +prometheus_http_request_duration_seconds_bucket{handler="/classic/static/*filepath",le="120"} 103 +prometheus_http_request_duration_seconds_bucket{handler="/classic/static/*filepath",le="+Inf"} 103 +prometheus_http_request_duration_seconds_sum{handler="/classic/static/*filepath"} 0.006113046000000001 +prometheus_http_request_duration_seconds_count{handler="/classic/static/*filepath"} 103 +prometheus_http_request_duration_seconds_bucket{handler="/config",le="0.1"} 13 +prometheus_http_request_duration_seconds_bucket{handler="/config",le="0.2"} 13 +prometheus_http_request_duration_seconds_bucket{handler="/config",le="0.4"} 13 +prometheus_http_request_duration_seconds_bucket{handler="/config",le="1"} 13 +prometheus_http_request_duration_seconds_bucket{handler="/config",le="3"} 13 +prometheus_http_request_duration_seconds_bucket{handler="/config",le="8"} 13 +prometheus_http_request_duration_seconds_bucket{handler="/config",le="20"} 13 +prometheus_http_request_duration_seconds_bucket{handler="/config",le="60"} 13 +prometheus_http_request_duration_seconds_bucket{handler="/config",le="120"} 13 +prometheus_http_request_duration_seconds_bucket{handler="/config",le="+Inf"} 13 +prometheus_http_request_duration_seconds_sum{handler="/config"} 0.0024490289999999997 +prometheus_http_request_duration_seconds_count{handler="/config"} 13 +prometheus_http_request_duration_seconds_bucket{handler="/consoles/*filepath",le="0.1"} 33 +prometheus_http_request_duration_seconds_bucket{handler="/consoles/*filepath",le="0.2"} 34 +prometheus_http_request_duration_seconds_bucket{handler="/consoles/*filepath",le="0.4"} 34 +prometheus_http_request_duration_seconds_bucket{handler="/consoles/*filepath",le="1"} 34 +prometheus_http_request_duration_seconds_bucket{handler="/consoles/*filepath",le="3"} 34 +prometheus_http_request_duration_seconds_bucket{handler="/consoles/*filepath",le="8"} 34 +prometheus_http_request_duration_seconds_bucket{handler="/consoles/*filepath",le="20"} 34 +prometheus_http_request_duration_seconds_bucket{handler="/consoles/*filepath",le="60"} 34 +prometheus_http_request_duration_seconds_bucket{handler="/consoles/*filepath",le="120"} 34 +prometheus_http_request_duration_seconds_bucket{handler="/consoles/*filepath",le="+Inf"} 34 +prometheus_http_request_duration_seconds_sum{handler="/consoles/*filepath"} 0.5689515199999999 +prometheus_http_request_duration_seconds_count{handler="/consoles/*filepath"} 34 +prometheus_http_request_duration_seconds_bucket{handler="/debug/*subpath",le="0.1"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/debug/*subpath",le="0.2"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/debug/*subpath",le="0.4"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/debug/*subpath",le="1"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/debug/*subpath",le="3"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/debug/*subpath",le="8"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/debug/*subpath",le="20"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/debug/*subpath",le="60"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/debug/*subpath",le="120"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/debug/*subpath",le="+Inf"} 4 +prometheus_http_request_duration_seconds_sum{handler="/debug/*subpath"} 0.086499352 +prometheus_http_request_duration_seconds_count{handler="/debug/*subpath"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/favicon.ico",le="0.1"} 177 +prometheus_http_request_duration_seconds_bucket{handler="/favicon.ico",le="0.2"} 177 +prometheus_http_request_duration_seconds_bucket{handler="/favicon.ico",le="0.4"} 177 +prometheus_http_request_duration_seconds_bucket{handler="/favicon.ico",le="1"} 177 +prometheus_http_request_duration_seconds_bucket{handler="/favicon.ico",le="3"} 177 +prometheus_http_request_duration_seconds_bucket{handler="/favicon.ico",le="8"} 177 +prometheus_http_request_duration_seconds_bucket{handler="/favicon.ico",le="20"} 177 +prometheus_http_request_duration_seconds_bucket{handler="/favicon.ico",le="60"} 177 +prometheus_http_request_duration_seconds_bucket{handler="/favicon.ico",le="120"} 177 +prometheus_http_request_duration_seconds_bucket{handler="/favicon.ico",le="+Inf"} 177 +prometheus_http_request_duration_seconds_sum{handler="/favicon.ico"} 0.05591882500000002 +prometheus_http_request_duration_seconds_count{handler="/favicon.ico"} 177 +prometheus_http_request_duration_seconds_bucket{handler="/favicon.svg",le="0.1"} 770 +prometheus_http_request_duration_seconds_bucket{handler="/favicon.svg",le="0.2"} 770 +prometheus_http_request_duration_seconds_bucket{handler="/favicon.svg",le="0.4"} 770 +prometheus_http_request_duration_seconds_bucket{handler="/favicon.svg",le="1"} 770 +prometheus_http_request_duration_seconds_bucket{handler="/favicon.svg",le="3"} 770 +prometheus_http_request_duration_seconds_bucket{handler="/favicon.svg",le="8"} 770 +prometheus_http_request_duration_seconds_bucket{handler="/favicon.svg",le="20"} 770 +prometheus_http_request_duration_seconds_bucket{handler="/favicon.svg",le="60"} 770 +prometheus_http_request_duration_seconds_bucket{handler="/favicon.svg",le="120"} 770 +prometheus_http_request_duration_seconds_bucket{handler="/favicon.svg",le="+Inf"} 770 +prometheus_http_request_duration_seconds_sum{handler="/favicon.svg"} 0.3058455699999999 +prometheus_http_request_duration_seconds_count{handler="/favicon.svg"} 770 +prometheus_http_request_duration_seconds_bucket{handler="/federate",le="0.1"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/federate",le="0.2"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/federate",le="0.4"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/federate",le="1"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/federate",le="3"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/federate",le="8"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/federate",le="20"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/federate",le="60"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/federate",le="120"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/federate",le="+Inf"} 4 +prometheus_http_request_duration_seconds_sum{handler="/federate"} 0.05996980699999999 +prometheus_http_request_duration_seconds_count{handler="/federate"} 4 +prometheus_http_request_duration_seconds_bucket{handler="/flags",le="0.1"} 9 +prometheus_http_request_duration_seconds_bucket{handler="/flags",le="0.2"} 9 +prometheus_http_request_duration_seconds_bucket{handler="/flags",le="0.4"} 9 +prometheus_http_request_duration_seconds_bucket{handler="/flags",le="1"} 9 +prometheus_http_request_duration_seconds_bucket{handler="/flags",le="3"} 9 +prometheus_http_request_duration_seconds_bucket{handler="/flags",le="8"} 9 +prometheus_http_request_duration_seconds_bucket{handler="/flags",le="20"} 9 +prometheus_http_request_duration_seconds_bucket{handler="/flags",le="60"} 9 +prometheus_http_request_duration_seconds_bucket{handler="/flags",le="120"} 9 +prometheus_http_request_duration_seconds_bucket{handler="/flags",le="+Inf"} 9 +prometheus_http_request_duration_seconds_sum{handler="/flags"} 0.001586781 +prometheus_http_request_duration_seconds_count{handler="/flags"} 9 +prometheus_http_request_duration_seconds_bucket{handler="/graph",le="0.1"} 751 +prometheus_http_request_duration_seconds_bucket{handler="/graph",le="0.2"} 751 +prometheus_http_request_duration_seconds_bucket{handler="/graph",le="0.4"} 751 +prometheus_http_request_duration_seconds_bucket{handler="/graph",le="1"} 751 +prometheus_http_request_duration_seconds_bucket{handler="/graph",le="3"} 751 +prometheus_http_request_duration_seconds_bucket{handler="/graph",le="8"} 751 +prometheus_http_request_duration_seconds_bucket{handler="/graph",le="20"} 751 +prometheus_http_request_duration_seconds_bucket{handler="/graph",le="60"} 751 +prometheus_http_request_duration_seconds_bucket{handler="/graph",le="120"} 751 +prometheus_http_request_duration_seconds_bucket{handler="/graph",le="+Inf"} 751 +prometheus_http_request_duration_seconds_sum{handler="/graph"} 0.032583203000000005 +prometheus_http_request_duration_seconds_count{handler="/graph"} 751 +prometheus_http_request_duration_seconds_bucket{handler="/metrics",le="0.1"} 3.988654e+06 +prometheus_http_request_duration_seconds_bucket{handler="/metrics",le="0.2"} 4.052461e+06 +prometheus_http_request_duration_seconds_bucket{handler="/metrics",le="0.4"} 4.057771e+06 +prometheus_http_request_duration_seconds_bucket{handler="/metrics",le="1"} 4.058809e+06 +prometheus_http_request_duration_seconds_bucket{handler="/metrics",le="3"} 4.059071e+06 +prometheus_http_request_duration_seconds_bucket{handler="/metrics",le="8"} 4.059092e+06 +prometheus_http_request_duration_seconds_bucket{handler="/metrics",le="20"} 4.059092e+06 +prometheus_http_request_duration_seconds_bucket{handler="/metrics",le="60"} 4.059092e+06 +prometheus_http_request_duration_seconds_bucket{handler="/metrics",le="120"} 4.059092e+06 +prometheus_http_request_duration_seconds_bucket{handler="/metrics",le="+Inf"} 4.059092e+06 +prometheus_http_request_duration_seconds_sum{handler="/metrics"} 107584.12561354278 +prometheus_http_request_duration_seconds_count{handler="/metrics"} 4.059092e+06 +prometheus_http_request_duration_seconds_bucket{handler="/query",le="0.1"} 1774 +prometheus_http_request_duration_seconds_bucket{handler="/query",le="0.2"} 1774 +prometheus_http_request_duration_seconds_bucket{handler="/query",le="0.4"} 1774 +prometheus_http_request_duration_seconds_bucket{handler="/query",le="1"} 1774 +prometheus_http_request_duration_seconds_bucket{handler="/query",le="3"} 1774 +prometheus_http_request_duration_seconds_bucket{handler="/query",le="8"} 1774 +prometheus_http_request_duration_seconds_bucket{handler="/query",le="20"} 1774 +prometheus_http_request_duration_seconds_bucket{handler="/query",le="60"} 1774 +prometheus_http_request_duration_seconds_bucket{handler="/query",le="120"} 1774 +prometheus_http_request_duration_seconds_bucket{handler="/query",le="+Inf"} 1774 +prometheus_http_request_duration_seconds_sum{handler="/query"} 0.5220201759999993 +prometheus_http_request_duration_seconds_count{handler="/query"} 1774 +prometheus_http_request_duration_seconds_bucket{handler="/rules",le="0.1"} 8672 +prometheus_http_request_duration_seconds_bucket{handler="/rules",le="0.2"} 8672 +prometheus_http_request_duration_seconds_bucket{handler="/rules",le="0.4"} 8673 +prometheus_http_request_duration_seconds_bucket{handler="/rules",le="1"} 8673 +prometheus_http_request_duration_seconds_bucket{handler="/rules",le="3"} 8673 +prometheus_http_request_duration_seconds_bucket{handler="/rules",le="8"} 8673 +prometheus_http_request_duration_seconds_bucket{handler="/rules",le="20"} 8673 +prometheus_http_request_duration_seconds_bucket{handler="/rules",le="60"} 8673 +prometheus_http_request_duration_seconds_bucket{handler="/rules",le="120"} 8673 +prometheus_http_request_duration_seconds_bucket{handler="/rules",le="+Inf"} 8673 +prometheus_http_request_duration_seconds_sum{handler="/rules"} 2.776021043000005 +prometheus_http_request_duration_seconds_count{handler="/rules"} 8673 +prometheus_http_request_duration_seconds_bucket{handler="/service-discovery",le="0.1"} 20 +prometheus_http_request_duration_seconds_bucket{handler="/service-discovery",le="0.2"} 20 +prometheus_http_request_duration_seconds_bucket{handler="/service-discovery",le="0.4"} 20 +prometheus_http_request_duration_seconds_bucket{handler="/service-discovery",le="1"} 20 +prometheus_http_request_duration_seconds_bucket{handler="/service-discovery",le="3"} 20 +prometheus_http_request_duration_seconds_bucket{handler="/service-discovery",le="8"} 20 +prometheus_http_request_duration_seconds_bucket{handler="/service-discovery",le="20"} 20 +prometheus_http_request_duration_seconds_bucket{handler="/service-discovery",le="60"} 20 +prometheus_http_request_duration_seconds_bucket{handler="/service-discovery",le="120"} 20 +prometheus_http_request_duration_seconds_bucket{handler="/service-discovery",le="+Inf"} 20 +prometheus_http_request_duration_seconds_sum{handler="/service-discovery"} 0.004057062 +prometheus_http_request_duration_seconds_count{handler="/service-discovery"} 20 +prometheus_http_request_duration_seconds_bucket{handler="/status",le="0.1"} 46 +prometheus_http_request_duration_seconds_bucket{handler="/status",le="0.2"} 46 +prometheus_http_request_duration_seconds_bucket{handler="/status",le="0.4"} 46 +prometheus_http_request_duration_seconds_bucket{handler="/status",le="1"} 46 +prometheus_http_request_duration_seconds_bucket{handler="/status",le="3"} 46 +prometheus_http_request_duration_seconds_bucket{handler="/status",le="8"} 46 +prometheus_http_request_duration_seconds_bucket{handler="/status",le="20"} 46 +prometheus_http_request_duration_seconds_bucket{handler="/status",le="60"} 46 +prometheus_http_request_duration_seconds_bucket{handler="/status",le="120"} 46 +prometheus_http_request_duration_seconds_bucket{handler="/status",le="+Inf"} 46 +prometheus_http_request_duration_seconds_sum{handler="/status"} 0.010107473 +prometheus_http_request_duration_seconds_count{handler="/status"} 46 +prometheus_http_request_duration_seconds_bucket{handler="/targets",le="0.1"} 39 +prometheus_http_request_duration_seconds_bucket{handler="/targets",le="0.2"} 39 +prometheus_http_request_duration_seconds_bucket{handler="/targets",le="0.4"} 39 +prometheus_http_request_duration_seconds_bucket{handler="/targets",le="1"} 39 +prometheus_http_request_duration_seconds_bucket{handler="/targets",le="3"} 39 +prometheus_http_request_duration_seconds_bucket{handler="/targets",le="8"} 39 +prometheus_http_request_duration_seconds_bucket{handler="/targets",le="20"} 39 +prometheus_http_request_duration_seconds_bucket{handler="/targets",le="60"} 39 +prometheus_http_request_duration_seconds_bucket{handler="/targets",le="120"} 39 +prometheus_http_request_duration_seconds_bucket{handler="/targets",le="+Inf"} 39 +prometheus_http_request_duration_seconds_sum{handler="/targets"} 0.009001180000000001 +prometheus_http_request_duration_seconds_count{handler="/targets"} 39 +prometheus_http_request_duration_seconds_bucket{handler="/tsdb-status",le="0.1"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/tsdb-status",le="0.2"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/tsdb-status",le="0.4"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/tsdb-status",le="1"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/tsdb-status",le="3"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/tsdb-status",le="8"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/tsdb-status",le="20"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/tsdb-status",le="60"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/tsdb-status",le="120"} 49 +prometheus_http_request_duration_seconds_bucket{handler="/tsdb-status",le="+Inf"} 49 +prometheus_http_request_duration_seconds_sum{handler="/tsdb-status"} 0.018204165 +prometheus_http_request_duration_seconds_count{handler="/tsdb-status"} 49 +# HELP prometheus_http_requests_total Counter of HTTP requests. +# TYPE prometheus_http_requests_total counter +prometheus_http_requests_total{code="200",handler="/"} 0 +prometheus_http_requests_total{code="200",handler="/-/healthy"} 29524 +prometheus_http_requests_total{code="200",handler="/-/quit"} 0 +prometheus_http_requests_total{code="200",handler="/-/ready"} 49 +prometheus_http_requests_total{code="200",handler="/-/reload"} 0 +prometheus_http_requests_total{code="200",handler="/alertmanager-discovery"} 0 +prometheus_http_requests_total{code="200",handler="/alerts"} 48 +prometheus_http_requests_total{code="200",handler="/api/v1/*path"} 0 +prometheus_http_requests_total{code="200",handler="/api/v1/admin/tsdb/clean_tombstones"} 0 +prometheus_http_requests_total{code="200",handler="/api/v1/admin/tsdb/delete_series"} 0 +prometheus_http_requests_total{code="200",handler="/api/v1/admin/tsdb/snapshot"} 0 +prometheus_http_requests_total{code="200",handler="/api/v1/alertmanagers"} 8 +prometheus_http_requests_total{code="200",handler="/api/v1/alerts"} 14635 +prometheus_http_requests_total{code="200",handler="/api/v1/format_query"} 3 +prometheus_http_requests_total{code="200",handler="/api/v1/label/:name/values"} 14679 +prometheus_http_requests_total{code="200",handler="/api/v1/labels"} 4915 +prometheus_http_requests_total{code="200",handler="/api/v1/metadata"} 2093 +prometheus_http_requests_total{code="200",handler="/api/v1/notifications"} 12 +prometheus_http_requests_total{code="200",handler="/api/v1/notifications/live"} 4529 +prometheus_http_requests_total{code="200",handler="/api/v1/otlp/v1/metrics"} 0 +prometheus_http_requests_total{code="200",handler="/api/v1/parse_query"} 247 +prometheus_http_requests_total{code="200",handler="/api/v1/query"} 326411 +prometheus_http_requests_total{code="200",handler="/api/v1/query_exemplars"} 841 +prometheus_http_requests_total{code="200",handler="/api/v1/query_range"} 1.183734e+06 +prometheus_http_requests_total{code="200",handler="/api/v1/read"} 16 +prometheus_http_requests_total{code="200",handler="/api/v1/rules"} 88658 +prometheus_http_requests_total{code="200",handler="/api/v1/scrape_pools"} 142 +prometheus_http_requests_total{code="200",handler="/api/v1/series"} 1235 +prometheus_http_requests_total{code="200",handler="/api/v1/status/buildinfo"} 4413 +prometheus_http_requests_total{code="200",handler="/api/v1/status/config"} 84722 +prometheus_http_requests_total{code="200",handler="/api/v1/status/flags"} 32 +prometheus_http_requests_total{code="200",handler="/api/v1/status/runtimeinfo"} 863 +prometheus_http_requests_total{code="200",handler="/api/v1/status/tsdb"} 94 +prometheus_http_requests_total{code="200",handler="/api/v1/status/walreplay"} 49 +prometheus_http_requests_total{code="200",handler="/api/v1/targets"} 191 +prometheus_http_requests_total{code="200",handler="/api/v1/targets/metadata"} 16 +prometheus_http_requests_total{code="200",handler="/api/v1/write"} 0 +prometheus_http_requests_total{code="200",handler="/assets/*filepath"} 2213 +prometheus_http_requests_total{code="200",handler="/classic/static/*filepath"} 0 +prometheus_http_requests_total{code="200",handler="/config"} 13 +prometheus_http_requests_total{code="200",handler="/consoles/*filepath"} 17 +prometheus_http_requests_total{code="200",handler="/debug/*subpath"} 3 +prometheus_http_requests_total{code="200",handler="/favicon.ico"} 0 +prometheus_http_requests_total{code="200",handler="/favicon.svg"} 769 +prometheus_http_requests_total{code="200",handler="/federate"} 4 +prometheus_http_requests_total{code="200",handler="/flags"} 9 +prometheus_http_requests_total{code="200",handler="/graph"} 0 +prometheus_http_requests_total{code="200",handler="/manifest.json"} 0 +prometheus_http_requests_total{code="200",handler="/metrics"} 4.059092e+06 +prometheus_http_requests_total{code="200",handler="/query"} 1774 +prometheus_http_requests_total{code="200",handler="/rules"} 8673 +prometheus_http_requests_total{code="200",handler="/service-discovery"} 20 +prometheus_http_requests_total{code="200",handler="/status"} 46 +prometheus_http_requests_total{code="200",handler="/targets"} 39 +prometheus_http_requests_total{code="200",handler="/tsdb-status"} 49 +prometheus_http_requests_total{code="200",handler="/version"} 0 +prometheus_http_requests_total{code="204",handler="/api/v1/*path"} 27 +prometheus_http_requests_total{code="204",handler="/api/v1/notifications/live"} 5 +prometheus_http_requests_total{code="301",handler="/debug/*subpath"} 1 +prometheus_http_requests_total{code="302",handler="/"} 688 +prometheus_http_requests_total{code="302",handler="/graph"} 751 +prometheus_http_requests_total{code="400",handler="/api/v1/format_query"} 1 +prometheus_http_requests_total{code="400",handler="/api/v1/label/:name/values"} 3292 +prometheus_http_requests_total{code="400",handler="/api/v1/labels"} 6 +prometheus_http_requests_total{code="400",handler="/api/v1/parse_query"} 18 +prometheus_http_requests_total{code="400",handler="/api/v1/query"} 155 +prometheus_http_requests_total{code="400",handler="/api/v1/query_range"} 263 +prometheus_http_requests_total{code="400",handler="/api/v1/rules"} 4 +prometheus_http_requests_total{code="400",handler="/api/v1/series"} 11 +prometheus_http_requests_total{code="400",handler="/api/v1/targets/metadata"} 2 +prometheus_http_requests_total{code="404",handler="/assets/*filepath"} 12 +prometheus_http_requests_total{code="404",handler="/classic/static/*filepath"} 103 +prometheus_http_requests_total{code="404",handler="/consoles/*filepath"} 17 +prometheus_http_requests_total{code="404",handler="/favicon.ico"} 177 +prometheus_http_requests_total{code="416",handler="/favicon.svg"} 1 +prometheus_http_requests_total{code="422",handler="/api/v1/query"} 114 +prometheus_http_requests_total{code="422",handler="/api/v1/query_range"} 31 +prometheus_http_requests_total{code="499",handler="/api/v1/query"} 7 +prometheus_http_requests_total{code="499",handler="/api/v1/query_range"} 20 +prometheus_http_requests_total{code="503",handler="/api/v1/query"} 7 +prometheus_http_requests_total{code="503",handler="/api/v1/query_range"} 4 +prometheus_http_requests_total{code="503",handler="/api/v1/status/config"} 1 +# HELP prometheus_http_response_size_bytes Histogram of response size for HTTP requests. +# TYPE prometheus_http_response_size_bytes histogram +prometheus_http_response_size_bytes_bucket{handler="/",le="100"} 688 +prometheus_http_response_size_bytes_bucket{handler="/",le="1000"} 688 +prometheus_http_response_size_bytes_bucket{handler="/",le="10000"} 688 +prometheus_http_response_size_bytes_bucket{handler="/",le="100000"} 688 +prometheus_http_response_size_bytes_bucket{handler="/",le="1e+06"} 688 +prometheus_http_response_size_bytes_bucket{handler="/",le="1e+07"} 688 +prometheus_http_response_size_bytes_bucket{handler="/",le="1e+08"} 688 +prometheus_http_response_size_bytes_bucket{handler="/",le="1e+09"} 688 +prometheus_http_response_size_bytes_bucket{handler="/",le="+Inf"} 688 +prometheus_http_response_size_bytes_sum{handler="/"} 19952 +prometheus_http_response_size_bytes_count{handler="/"} 688 +prometheus_http_response_size_bytes_bucket{handler="/-/healthy",le="100"} 29524 +prometheus_http_response_size_bytes_bucket{handler="/-/healthy",le="1000"} 29524 +prometheus_http_response_size_bytes_bucket{handler="/-/healthy",le="10000"} 29524 +prometheus_http_response_size_bytes_bucket{handler="/-/healthy",le="100000"} 29524 +prometheus_http_response_size_bytes_bucket{handler="/-/healthy",le="1e+06"} 29524 +prometheus_http_response_size_bytes_bucket{handler="/-/healthy",le="1e+07"} 29524 +prometheus_http_response_size_bytes_bucket{handler="/-/healthy",le="1e+08"} 29524 +prometheus_http_response_size_bytes_bucket{handler="/-/healthy",le="1e+09"} 29524 +prometheus_http_response_size_bytes_bucket{handler="/-/healthy",le="+Inf"} 29524 +prometheus_http_response_size_bytes_sum{handler="/-/healthy"} 885720 +prometheus_http_response_size_bytes_count{handler="/-/healthy"} 29524 +prometheus_http_response_size_bytes_bucket{handler="/-/ready",le="100"} 49 +prometheus_http_response_size_bytes_bucket{handler="/-/ready",le="1000"} 49 +prometheus_http_response_size_bytes_bucket{handler="/-/ready",le="10000"} 49 +prometheus_http_response_size_bytes_bucket{handler="/-/ready",le="100000"} 49 +prometheus_http_response_size_bytes_bucket{handler="/-/ready",le="1e+06"} 49 +prometheus_http_response_size_bytes_bucket{handler="/-/ready",le="1e+07"} 49 +prometheus_http_response_size_bytes_bucket{handler="/-/ready",le="1e+08"} 49 +prometheus_http_response_size_bytes_bucket{handler="/-/ready",le="1e+09"} 49 +prometheus_http_response_size_bytes_bucket{handler="/-/ready",le="+Inf"} 49 +prometheus_http_response_size_bytes_sum{handler="/-/ready"} 1372 +prometheus_http_response_size_bytes_count{handler="/-/ready"} 49 +prometheus_http_response_size_bytes_bucket{handler="/alerts",le="100"} 0 +prometheus_http_response_size_bytes_bucket{handler="/alerts",le="1000"} 0 +prometheus_http_response_size_bytes_bucket{handler="/alerts",le="10000"} 48 +prometheus_http_response_size_bytes_bucket{handler="/alerts",le="100000"} 48 +prometheus_http_response_size_bytes_bucket{handler="/alerts",le="1e+06"} 48 +prometheus_http_response_size_bytes_bucket{handler="/alerts",le="1e+07"} 48 +prometheus_http_response_size_bytes_bucket{handler="/alerts",le="1e+08"} 48 +prometheus_http_response_size_bytes_bucket{handler="/alerts",le="1e+09"} 48 +prometheus_http_response_size_bytes_bucket{handler="/alerts",le="+Inf"} 48 +prometheus_http_response_size_bytes_sum{handler="/alerts"} 84096 +prometheus_http_response_size_bytes_count{handler="/alerts"} 48 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/*path",le="100"} 27 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/*path",le="1000"} 27 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/*path",le="10000"} 27 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/*path",le="100000"} 27 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/*path",le="1e+06"} 27 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/*path",le="1e+07"} 27 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/*path",le="1e+08"} 27 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/*path",le="1e+09"} 27 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/*path",le="+Inf"} 27 +prometheus_http_response_size_bytes_sum{handler="/api/v1/*path"} 0 +prometheus_http_response_size_bytes_count{handler="/api/v1/*path"} 27 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/alertmanagers",le="100"} 0 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/alertmanagers",le="1000"} 8 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/alertmanagers",le="10000"} 8 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/alertmanagers",le="100000"} 8 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/alertmanagers",le="1e+06"} 8 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/alertmanagers",le="1e+07"} 8 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/alertmanagers",le="1e+08"} 8 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/alertmanagers",le="1e+09"} 8 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/alertmanagers",le="+Inf"} 8 +prometheus_http_response_size_bytes_sum{handler="/api/v1/alertmanagers"} 1064 +prometheus_http_response_size_bytes_count{handler="/api/v1/alertmanagers"} 8 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/alerts",le="100"} 0 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/alerts",le="1000"} 14635 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/alerts",le="10000"} 14635 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/alerts",le="100000"} 14635 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/alerts",le="1e+06"} 14635 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/alerts",le="1e+07"} 14635 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/alerts",le="1e+08"} 14635 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/alerts",le="1e+09"} 14635 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/alerts",le="+Inf"} 14635 +prometheus_http_response_size_bytes_sum{handler="/api/v1/alerts"} 1.0260926e+07 +prometheus_http_response_size_bytes_count{handler="/api/v1/alerts"} 14635 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/format_query",le="100"} 0 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/format_query",le="1000"} 4 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/format_query",le="10000"} 4 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/format_query",le="100000"} 4 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/format_query",le="1e+06"} 4 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/format_query",le="1e+07"} 4 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/format_query",le="1e+08"} 4 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/format_query",le="1e+09"} 4 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/format_query",le="+Inf"} 4 +prometheus_http_response_size_bytes_sum{handler="/api/v1/format_query"} 495 +prometheus_http_response_size_bytes_count{handler="/api/v1/format_query"} 4 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/label/:name/values",le="100"} 8167 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/label/:name/values",le="1000"} 15939 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/label/:name/values",le="10000"} 17901 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/label/:name/values",le="100000"} 17971 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/label/:name/values",le="1e+06"} 17971 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/label/:name/values",le="1e+07"} 17971 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/label/:name/values",le="1e+08"} 17971 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/label/:name/values",le="1e+09"} 17971 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/label/:name/values",le="+Inf"} 17971 +prometheus_http_response_size_bytes_sum{handler="/api/v1/label/:name/values"} 2.0009454e+07 +prometheus_http_response_size_bytes_count{handler="/api/v1/label/:name/values"} 17971 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/labels",le="100"} 102 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/labels",le="1000"} 4908 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/labels",le="10000"} 4921 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/labels",le="100000"} 4921 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/labels",le="1e+06"} 4921 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/labels",le="1e+07"} 4921 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/labels",le="1e+08"} 4921 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/labels",le="1e+09"} 4921 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/labels",le="+Inf"} 4921 +prometheus_http_response_size_bytes_sum{handler="/api/v1/labels"} 3.16046e+06 +prometheus_http_response_size_bytes_count{handler="/api/v1/labels"} 4921 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/metadata",le="100"} 170 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/metadata",le="1000"} 1368 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/metadata",le="10000"} 1368 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/metadata",le="100000"} 2093 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/metadata",le="1e+06"} 2093 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/metadata",le="1e+07"} 2093 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/metadata",le="1e+08"} 2093 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/metadata",le="1e+09"} 2093 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/metadata",le="+Inf"} 2093 +prometheus_http_response_size_bytes_sum{handler="/api/v1/metadata"} 1.5950362e+07 +prometheus_http_response_size_bytes_count{handler="/api/v1/metadata"} 2093 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/notifications",le="100"} 12 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/notifications",le="1000"} 12 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/notifications",le="10000"} 12 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/notifications",le="100000"} 12 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/notifications",le="1e+06"} 12 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/notifications",le="1e+07"} 12 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/notifications",le="1e+08"} 12 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/notifications",le="1e+09"} 12 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/notifications",le="+Inf"} 12 +prometheus_http_response_size_bytes_sum{handler="/api/v1/notifications"} 360 +prometheus_http_response_size_bytes_count{handler="/api/v1/notifications"} 12 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/notifications/live",le="100"} 4529 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/notifications/live",le="1000"} 4534 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/notifications/live",le="10000"} 4534 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/notifications/live",le="100000"} 4534 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/notifications/live",le="1e+06"} 4534 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/notifications/live",le="1e+07"} 4534 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/notifications/live",le="1e+08"} 4534 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/notifications/live",le="1e+09"} 4534 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/notifications/live",le="+Inf"} 4534 +prometheus_http_response_size_bytes_sum{handler="/api/v1/notifications/live"} 1365 +prometheus_http_response_size_bytes_count{handler="/api/v1/notifications/live"} 4534 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/parse_query",le="100"} 0 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/parse_query",le="1000"} 265 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/parse_query",le="10000"} 265 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/parse_query",le="100000"} 265 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/parse_query",le="1e+06"} 265 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/parse_query",le="1e+07"} 265 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/parse_query",le="1e+08"} 265 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/parse_query",le="1e+09"} 265 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/parse_query",le="+Inf"} 265 +prometheus_http_response_size_bytes_sum{handler="/api/v1/parse_query"} 67620 +prometheus_http_response_size_bytes_count{handler="/api/v1/parse_query"} 265 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query",le="100"} 25147 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query",le="1000"} 325005 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query",le="10000"} 325143 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query",le="100000"} 326635 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query",le="1e+06"} 326692 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query",le="1e+07"} 326694 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query",le="1e+08"} 326694 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query",le="1e+09"} 326694 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query",le="+Inf"} 326694 +prometheus_http_response_size_bytes_sum{handler="/api/v1/query"} 2.34481756e+08 +prometheus_http_response_size_bytes_count{handler="/api/v1/query"} 326694 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query_exemplars",le="100"} 841 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query_exemplars",le="1000"} 841 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query_exemplars",le="10000"} 841 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query_exemplars",le="100000"} 841 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query_exemplars",le="1e+06"} 841 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query_exemplars",le="1e+07"} 841 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query_exemplars",le="1e+08"} 841 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query_exemplars",le="1e+09"} 841 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query_exemplars",le="+Inf"} 841 +prometheus_http_response_size_bytes_sum{handler="/api/v1/query_exemplars"} 50460 +prometheus_http_response_size_bytes_count{handler="/api/v1/query_exemplars"} 841 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query_range",le="100"} 69552 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query_range",le="1000"} 917165 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query_range",le="10000"} 1.152103e+06 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query_range",le="100000"} 1.183391e+06 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query_range",le="1e+06"} 1.183621e+06 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query_range",le="1e+07"} 1.184052e+06 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query_range",le="1e+08"} 1.184052e+06 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query_range",le="1e+09"} 1.184052e+06 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/query_range",le="+Inf"} 1.184052e+06 +prometheus_http_response_size_bytes_sum{handler="/api/v1/query_range"} 1.869092376e+09 +prometheus_http_response_size_bytes_count{handler="/api/v1/query_range"} 1.184052e+06 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/read",le="100"} 0 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/read",le="1000"} 0 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/read",le="10000"} 16 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/read",le="100000"} 16 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/read",le="1e+06"} 16 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/read",le="1e+07"} 16 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/read",le="1e+08"} 16 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/read",le="1e+09"} 16 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/read",le="+Inf"} 16 +prometheus_http_response_size_bytes_sum{handler="/api/v1/read"} 50800 +prometheus_http_response_size_bytes_count{handler="/api/v1/read"} 16 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/rules",le="100"} 0 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/rules",le="1000"} 4 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/rules",le="10000"} 88656 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/rules",le="100000"} 88662 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/rules",le="1e+06"} 88662 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/rules",le="1e+07"} 88662 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/rules",le="1e+08"} 88662 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/rules",le="1e+09"} 88662 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/rules",le="+Inf"} 88662 +prometheus_http_response_size_bytes_sum{handler="/api/v1/rules"} 7.01925022e+08 +prometheus_http_response_size_bytes_count{handler="/api/v1/rules"} 88662 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/scrape_pools",le="100"} 0 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/scrape_pools",le="1000"} 142 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/scrape_pools",le="10000"} 142 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/scrape_pools",le="100000"} 142 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/scrape_pools",le="1e+06"} 142 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/scrape_pools",le="1e+07"} 142 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/scrape_pools",le="1e+08"} 142 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/scrape_pools",le="1e+09"} 142 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/scrape_pools",le="+Inf"} 142 +prometheus_http_response_size_bytes_sum{handler="/api/v1/scrape_pools"} 18318 +prometheus_http_response_size_bytes_count{handler="/api/v1/scrape_pools"} 142 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/series",le="100"} 1011 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/series",le="1000"} 1233 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/series",le="10000"} 1244 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/series",le="100000"} 1246 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/series",le="1e+06"} 1246 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/series",le="1e+07"} 1246 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/series",le="1e+08"} 1246 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/series",le="1e+09"} 1246 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/series",le="+Inf"} 1246 +prometheus_http_response_size_bytes_sum{handler="/api/v1/series"} 235829 +prometheus_http_response_size_bytes_count{handler="/api/v1/series"} 1246 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/buildinfo",le="100"} 0 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/buildinfo",le="1000"} 4413 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/buildinfo",le="10000"} 4413 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/buildinfo",le="100000"} 4413 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/buildinfo",le="1e+06"} 4413 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/buildinfo",le="1e+07"} 4413 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/buildinfo",le="1e+08"} 4413 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/buildinfo",le="1e+09"} 4413 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/buildinfo",le="+Inf"} 4413 +prometheus_http_response_size_bytes_sum{handler="/api/v1/status/buildinfo"} 807579 +prometheus_http_response_size_bytes_count{handler="/api/v1/status/buildinfo"} 4413 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/config",le="100"} 1 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/config",le="1000"} 84723 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/config",le="10000"} 84723 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/config",le="100000"} 84723 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/config",le="1e+06"} 84723 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/config",le="1e+07"} 84723 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/config",le="1e+08"} 84723 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/config",le="1e+09"} 84723 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/config",le="+Inf"} 84723 +prometheus_http_response_size_bytes_sum{handler="/api/v1/status/config"} 6.3710963e+07 +prometheus_http_response_size_bytes_count{handler="/api/v1/status/config"} 84723 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/flags",le="100"} 0 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/flags",le="1000"} 32 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/flags",le="10000"} 32 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/flags",le="100000"} 32 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/flags",le="1e+06"} 32 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/flags",le="1e+07"} 32 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/flags",le="1e+08"} 32 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/flags",le="1e+09"} 32 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/flags",le="+Inf"} 32 +prometheus_http_response_size_bytes_sum{handler="/api/v1/status/flags"} 31968 +prometheus_http_response_size_bytes_count{handler="/api/v1/status/flags"} 32 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/runtimeinfo",le="100"} 0 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/runtimeinfo",le="1000"} 863 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/runtimeinfo",le="10000"} 863 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/runtimeinfo",le="100000"} 863 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/runtimeinfo",le="1e+06"} 863 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/runtimeinfo",le="1e+07"} 863 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/runtimeinfo",le="1e+08"} 863 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/runtimeinfo",le="1e+09"} 863 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/runtimeinfo",le="+Inf"} 863 +prometheus_http_response_size_bytes_sum{handler="/api/v1/status/runtimeinfo"} 240400 +prometheus_http_response_size_bytes_count{handler="/api/v1/status/runtimeinfo"} 863 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/tsdb",le="100"} 0 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/tsdb",le="1000"} 92 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/tsdb",le="10000"} 94 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/tsdb",le="100000"} 94 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/tsdb",le="1e+06"} 94 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/tsdb",le="1e+07"} 94 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/tsdb",le="1e+08"} 94 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/tsdb",le="1e+09"} 94 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/tsdb",le="+Inf"} 94 +prometheus_http_response_size_bytes_sum{handler="/api/v1/status/tsdb"} 66441 +prometheus_http_response_size_bytes_count{handler="/api/v1/status/tsdb"} 94 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/walreplay",le="100"} 49 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/walreplay",le="1000"} 49 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/walreplay",le="10000"} 49 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/walreplay",le="100000"} 49 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/walreplay",le="1e+06"} 49 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/walreplay",le="1e+07"} 49 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/walreplay",le="1e+08"} 49 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/walreplay",le="1e+09"} 49 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/status/walreplay",le="+Inf"} 49 +prometheus_http_response_size_bytes_sum{handler="/api/v1/status/walreplay"} 3381 +prometheus_http_response_size_bytes_count{handler="/api/v1/status/walreplay"} 49 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/targets",le="100"} 0 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/targets",le="1000"} 184 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/targets",le="10000"} 191 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/targets",le="100000"} 191 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/targets",le="1e+06"} 191 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/targets",le="1e+07"} 191 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/targets",le="1e+08"} 191 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/targets",le="1e+09"} 191 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/targets",le="+Inf"} 191 +prometheus_http_response_size_bytes_sum{handler="/api/v1/targets"} 191150 +prometheus_http_response_size_bytes_count{handler="/api/v1/targets"} 191 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/targets/metadata",le="100"} 1 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/targets/metadata",le="1000"} 18 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/targets/metadata",le="10000"} 18 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/targets/metadata",le="100000"} 18 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/targets/metadata",le="1e+06"} 18 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/targets/metadata",le="1e+07"} 18 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/targets/metadata",le="1e+08"} 18 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/targets/metadata",le="1e+09"} 18 +prometheus_http_response_size_bytes_bucket{handler="/api/v1/targets/metadata",le="+Inf"} 18 +prometheus_http_response_size_bytes_sum{handler="/api/v1/targets/metadata"} 2736 +prometheus_http_response_size_bytes_count{handler="/api/v1/targets/metadata"} 18 +prometheus_http_response_size_bytes_bucket{handler="/assets/*filepath",le="100"} 12 +prometheus_http_response_size_bytes_bucket{handler="/assets/*filepath",le="1000"} 12 +prometheus_http_response_size_bytes_bucket{handler="/assets/*filepath",le="10000"} 13 +prometheus_http_response_size_bytes_bucket{handler="/assets/*filepath",le="100000"} 142 +prometheus_http_response_size_bytes_bucket{handler="/assets/*filepath",le="1e+06"} 958 +prometheus_http_response_size_bytes_bucket{handler="/assets/*filepath",le="1e+07"} 2225 +prometheus_http_response_size_bytes_bucket{handler="/assets/*filepath",le="1e+08"} 2225 +prometheus_http_response_size_bytes_bucket{handler="/assets/*filepath",le="1e+09"} 2225 +prometheus_http_response_size_bytes_bucket{handler="/assets/*filepath",le="+Inf"} 2225 +prometheus_http_response_size_bytes_sum{handler="/assets/*filepath"} 3.525958804e+09 +prometheus_http_response_size_bytes_count{handler="/assets/*filepath"} 2225 +prometheus_http_response_size_bytes_bucket{handler="/classic/static/*filepath",le="100"} 103 +prometheus_http_response_size_bytes_bucket{handler="/classic/static/*filepath",le="1000"} 103 +prometheus_http_response_size_bytes_bucket{handler="/classic/static/*filepath",le="10000"} 103 +prometheus_http_response_size_bytes_bucket{handler="/classic/static/*filepath",le="100000"} 103 +prometheus_http_response_size_bytes_bucket{handler="/classic/static/*filepath",le="1e+06"} 103 +prometheus_http_response_size_bytes_bucket{handler="/classic/static/*filepath",le="1e+07"} 103 +prometheus_http_response_size_bytes_bucket{handler="/classic/static/*filepath",le="1e+08"} 103 +prometheus_http_response_size_bytes_bucket{handler="/classic/static/*filepath",le="1e+09"} 103 +prometheus_http_response_size_bytes_bucket{handler="/classic/static/*filepath",le="+Inf"} 103 +prometheus_http_response_size_bytes_sum{handler="/classic/static/*filepath"} 1957 +prometheus_http_response_size_bytes_count{handler="/classic/static/*filepath"} 103 +prometheus_http_response_size_bytes_bucket{handler="/config",le="100"} 0 +prometheus_http_response_size_bytes_bucket{handler="/config",le="1000"} 0 +prometheus_http_response_size_bytes_bucket{handler="/config",le="10000"} 13 +prometheus_http_response_size_bytes_bucket{handler="/config",le="100000"} 13 +prometheus_http_response_size_bytes_bucket{handler="/config",le="1e+06"} 13 +prometheus_http_response_size_bytes_bucket{handler="/config",le="1e+07"} 13 +prometheus_http_response_size_bytes_bucket{handler="/config",le="1e+08"} 13 +prometheus_http_response_size_bytes_bucket{handler="/config",le="1e+09"} 13 +prometheus_http_response_size_bytes_bucket{handler="/config",le="+Inf"} 13 +prometheus_http_response_size_bytes_sum{handler="/config"} 22776 +prometheus_http_response_size_bytes_count{handler="/config"} 13 +prometheus_http_response_size_bytes_bucket{handler="/consoles/*filepath",le="100"} 17 +prometheus_http_response_size_bytes_bucket{handler="/consoles/*filepath",le="1000"} 17 +prometheus_http_response_size_bytes_bucket{handler="/consoles/*filepath",le="10000"} 31 +prometheus_http_response_size_bytes_bucket{handler="/consoles/*filepath",le="100000"} 34 +prometheus_http_response_size_bytes_bucket{handler="/consoles/*filepath",le="1e+06"} 34 +prometheus_http_response_size_bytes_bucket{handler="/consoles/*filepath",le="1e+07"} 34 +prometheus_http_response_size_bytes_bucket{handler="/consoles/*filepath",le="1e+08"} 34 +prometheus_http_response_size_bytes_bucket{handler="/consoles/*filepath",le="1e+09"} 34 +prometheus_http_response_size_bytes_bucket{handler="/consoles/*filepath",le="+Inf"} 34 +prometheus_http_response_size_bytes_sum{handler="/consoles/*filepath"} 175556 +prometheus_http_response_size_bytes_count{handler="/consoles/*filepath"} 34 +prometheus_http_response_size_bytes_bucket{handler="/debug/*subpath",le="100"} 1 +prometheus_http_response_size_bytes_bucket{handler="/debug/*subpath",le="1000"} 1 +prometheus_http_response_size_bytes_bucket{handler="/debug/*subpath",le="10000"} 4 +prometheus_http_response_size_bytes_bucket{handler="/debug/*subpath",le="100000"} 4 +prometheus_http_response_size_bytes_bucket{handler="/debug/*subpath",le="1e+06"} 4 +prometheus_http_response_size_bytes_bucket{handler="/debug/*subpath",le="1e+07"} 4 +prometheus_http_response_size_bytes_bucket{handler="/debug/*subpath",le="1e+08"} 4 +prometheus_http_response_size_bytes_bucket{handler="/debug/*subpath",le="1e+09"} 4 +prometheus_http_response_size_bytes_bucket{handler="/debug/*subpath",le="+Inf"} 4 +prometheus_http_response_size_bytes_sum{handler="/debug/*subpath"} 7062 +prometheus_http_response_size_bytes_count{handler="/debug/*subpath"} 4 +prometheus_http_response_size_bytes_bucket{handler="/favicon.ico",le="100"} 177 +prometheus_http_response_size_bytes_bucket{handler="/favicon.ico",le="1000"} 177 +prometheus_http_response_size_bytes_bucket{handler="/favicon.ico",le="10000"} 177 +prometheus_http_response_size_bytes_bucket{handler="/favicon.ico",le="100000"} 177 +prometheus_http_response_size_bytes_bucket{handler="/favicon.ico",le="1e+06"} 177 +prometheus_http_response_size_bytes_bucket{handler="/favicon.ico",le="1e+07"} 177 +prometheus_http_response_size_bytes_bucket{handler="/favicon.ico",le="1e+08"} 177 +prometheus_http_response_size_bytes_bucket{handler="/favicon.ico",le="1e+09"} 177 +prometheus_http_response_size_bytes_bucket{handler="/favicon.ico",le="+Inf"} 177 +prometheus_http_response_size_bytes_sum{handler="/favicon.ico"} 3363 +prometheus_http_response_size_bytes_count{handler="/favicon.ico"} 177 +prometheus_http_response_size_bytes_bucket{handler="/favicon.svg",le="100"} 1 +prometheus_http_response_size_bytes_bucket{handler="/favicon.svg",le="1000"} 1 +prometheus_http_response_size_bytes_bucket{handler="/favicon.svg",le="10000"} 770 +prometheus_http_response_size_bytes_bucket{handler="/favicon.svg",le="100000"} 770 +prometheus_http_response_size_bytes_bucket{handler="/favicon.svg",le="1e+06"} 770 +prometheus_http_response_size_bytes_bucket{handler="/favicon.svg",le="1e+07"} 770 +prometheus_http_response_size_bytes_bucket{handler="/favicon.svg",le="1e+08"} 770 +prometheus_http_response_size_bytes_bucket{handler="/favicon.svg",le="1e+09"} 770 +prometheus_http_response_size_bytes_bucket{handler="/favicon.svg",le="+Inf"} 770 +prometheus_http_response_size_bytes_sum{handler="/favicon.svg"} 2.135541e+06 +prometheus_http_response_size_bytes_count{handler="/favicon.svg"} 770 +prometheus_http_response_size_bytes_bucket{handler="/federate",le="100"} 2 +prometheus_http_response_size_bytes_bucket{handler="/federate",le="1000"} 2 +prometheus_http_response_size_bytes_bucket{handler="/federate",le="10000"} 2 +prometheus_http_response_size_bytes_bucket{handler="/federate",le="100000"} 4 +prometheus_http_response_size_bytes_bucket{handler="/federate",le="1e+06"} 4 +prometheus_http_response_size_bytes_bucket{handler="/federate",le="1e+07"} 4 +prometheus_http_response_size_bytes_bucket{handler="/federate",le="1e+08"} 4 +prometheus_http_response_size_bytes_bucket{handler="/federate",le="1e+09"} 4 +prometheus_http_response_size_bytes_bucket{handler="/federate",le="+Inf"} 4 +prometheus_http_response_size_bytes_sum{handler="/federate"} 28439 +prometheus_http_response_size_bytes_count{handler="/federate"} 4 +prometheus_http_response_size_bytes_bucket{handler="/flags",le="100"} 0 +prometheus_http_response_size_bytes_bucket{handler="/flags",le="1000"} 0 +prometheus_http_response_size_bytes_bucket{handler="/flags",le="10000"} 9 +prometheus_http_response_size_bytes_bucket{handler="/flags",le="100000"} 9 +prometheus_http_response_size_bytes_bucket{handler="/flags",le="1e+06"} 9 +prometheus_http_response_size_bytes_bucket{handler="/flags",le="1e+07"} 9 +prometheus_http_response_size_bytes_bucket{handler="/flags",le="1e+08"} 9 +prometheus_http_response_size_bytes_bucket{handler="/flags",le="1e+09"} 9 +prometheus_http_response_size_bytes_bucket{handler="/flags",le="+Inf"} 9 +prometheus_http_response_size_bytes_sum{handler="/flags"} 15768 +prometheus_http_response_size_bytes_count{handler="/flags"} 9 +prometheus_http_response_size_bytes_bucket{handler="/graph",le="100"} 130 +prometheus_http_response_size_bytes_bucket{handler="/graph",le="1000"} 744 +prometheus_http_response_size_bytes_bucket{handler="/graph",le="10000"} 751 +prometheus_http_response_size_bytes_bucket{handler="/graph",le="100000"} 751 +prometheus_http_response_size_bytes_bucket{handler="/graph",le="1e+06"} 751 +prometheus_http_response_size_bytes_bucket{handler="/graph",le="1e+07"} 751 +prometheus_http_response_size_bytes_bucket{handler="/graph",le="1e+08"} 751 +prometheus_http_response_size_bytes_bucket{handler="/graph",le="1e+09"} 751 +prometheus_http_response_size_bytes_bucket{handler="/graph",le="+Inf"} 751 +prometheus_http_response_size_bytes_sum{handler="/graph"} 164775 +prometheus_http_response_size_bytes_count{handler="/graph"} 751 +prometheus_http_response_size_bytes_bucket{handler="/metrics",le="100"} 0 +prometheus_http_response_size_bytes_bucket{handler="/metrics",le="1000"} 0 +prometheus_http_response_size_bytes_bucket{handler="/metrics",le="10000"} 0 +prometheus_http_response_size_bytes_bucket{handler="/metrics",le="100000"} 4.059088e+06 +prometheus_http_response_size_bytes_bucket{handler="/metrics",le="1e+06"} 4.059092e+06 +prometheus_http_response_size_bytes_bucket{handler="/metrics",le="1e+07"} 4.059092e+06 +prometheus_http_response_size_bytes_bucket{handler="/metrics",le="1e+08"} 4.059092e+06 +prometheus_http_response_size_bytes_bucket{handler="/metrics",le="1e+09"} 4.059092e+06 +prometheus_http_response_size_bytes_bucket{handler="/metrics",le="+Inf"} 4.059092e+06 +prometheus_http_response_size_bytes_sum{handler="/metrics"} 8.2515439707e+10 +prometheus_http_response_size_bytes_count{handler="/metrics"} 4.059092e+06 +prometheus_http_response_size_bytes_bucket{handler="/query",le="100"} 0 +prometheus_http_response_size_bytes_bucket{handler="/query",le="1000"} 0 +prometheus_http_response_size_bytes_bucket{handler="/query",le="10000"} 1774 +prometheus_http_response_size_bytes_bucket{handler="/query",le="100000"} 1774 +prometheus_http_response_size_bytes_bucket{handler="/query",le="1e+06"} 1774 +prometheus_http_response_size_bytes_bucket{handler="/query",le="1e+07"} 1774 +prometheus_http_response_size_bytes_bucket{handler="/query",le="1e+08"} 1774 +prometheus_http_response_size_bytes_bucket{handler="/query",le="1e+09"} 1774 +prometheus_http_response_size_bytes_bucket{handler="/query",le="+Inf"} 1774 +prometheus_http_response_size_bytes_sum{handler="/query"} 3.108048e+06 +prometheus_http_response_size_bytes_count{handler="/query"} 1774 +prometheus_http_response_size_bytes_bucket{handler="/rules",le="100"} 0 +prometheus_http_response_size_bytes_bucket{handler="/rules",le="1000"} 0 +prometheus_http_response_size_bytes_bucket{handler="/rules",le="10000"} 8673 +prometheus_http_response_size_bytes_bucket{handler="/rules",le="100000"} 8673 +prometheus_http_response_size_bytes_bucket{handler="/rules",le="1e+06"} 8673 +prometheus_http_response_size_bytes_bucket{handler="/rules",le="1e+07"} 8673 +prometheus_http_response_size_bytes_bucket{handler="/rules",le="1e+08"} 8673 +prometheus_http_response_size_bytes_bucket{handler="/rules",le="1e+09"} 8673 +prometheus_http_response_size_bytes_bucket{handler="/rules",le="+Inf"} 8673 +prometheus_http_response_size_bytes_sum{handler="/rules"} 1.5195096e+07 +prometheus_http_response_size_bytes_count{handler="/rules"} 8673 +prometheus_http_response_size_bytes_bucket{handler="/service-discovery",le="100"} 0 +prometheus_http_response_size_bytes_bucket{handler="/service-discovery",le="1000"} 0 +prometheus_http_response_size_bytes_bucket{handler="/service-discovery",le="10000"} 20 +prometheus_http_response_size_bytes_bucket{handler="/service-discovery",le="100000"} 20 +prometheus_http_response_size_bytes_bucket{handler="/service-discovery",le="1e+06"} 20 +prometheus_http_response_size_bytes_bucket{handler="/service-discovery",le="1e+07"} 20 +prometheus_http_response_size_bytes_bucket{handler="/service-discovery",le="1e+08"} 20 +prometheus_http_response_size_bytes_bucket{handler="/service-discovery",le="1e+09"} 20 +prometheus_http_response_size_bytes_bucket{handler="/service-discovery",le="+Inf"} 20 +prometheus_http_response_size_bytes_sum{handler="/service-discovery"} 35040 +prometheus_http_response_size_bytes_count{handler="/service-discovery"} 20 +prometheus_http_response_size_bytes_bucket{handler="/status",le="100"} 0 +prometheus_http_response_size_bytes_bucket{handler="/status",le="1000"} 0 +prometheus_http_response_size_bytes_bucket{handler="/status",le="10000"} 46 +prometheus_http_response_size_bytes_bucket{handler="/status",le="100000"} 46 +prometheus_http_response_size_bytes_bucket{handler="/status",le="1e+06"} 46 +prometheus_http_response_size_bytes_bucket{handler="/status",le="1e+07"} 46 +prometheus_http_response_size_bytes_bucket{handler="/status",le="1e+08"} 46 +prometheus_http_response_size_bytes_bucket{handler="/status",le="1e+09"} 46 +prometheus_http_response_size_bytes_bucket{handler="/status",le="+Inf"} 46 +prometheus_http_response_size_bytes_sum{handler="/status"} 80592 +prometheus_http_response_size_bytes_count{handler="/status"} 46 +prometheus_http_response_size_bytes_bucket{handler="/targets",le="100"} 0 +prometheus_http_response_size_bytes_bucket{handler="/targets",le="1000"} 0 +prometheus_http_response_size_bytes_bucket{handler="/targets",le="10000"} 39 +prometheus_http_response_size_bytes_bucket{handler="/targets",le="100000"} 39 +prometheus_http_response_size_bytes_bucket{handler="/targets",le="1e+06"} 39 +prometheus_http_response_size_bytes_bucket{handler="/targets",le="1e+07"} 39 +prometheus_http_response_size_bytes_bucket{handler="/targets",le="1e+08"} 39 +prometheus_http_response_size_bytes_bucket{handler="/targets",le="1e+09"} 39 +prometheus_http_response_size_bytes_bucket{handler="/targets",le="+Inf"} 39 +prometheus_http_response_size_bytes_sum{handler="/targets"} 68328 +prometheus_http_response_size_bytes_count{handler="/targets"} 39 +prometheus_http_response_size_bytes_bucket{handler="/tsdb-status",le="100"} 0 +prometheus_http_response_size_bytes_bucket{handler="/tsdb-status",le="1000"} 0 +prometheus_http_response_size_bytes_bucket{handler="/tsdb-status",le="10000"} 49 +prometheus_http_response_size_bytes_bucket{handler="/tsdb-status",le="100000"} 49 +prometheus_http_response_size_bytes_bucket{handler="/tsdb-status",le="1e+06"} 49 +prometheus_http_response_size_bytes_bucket{handler="/tsdb-status",le="1e+07"} 49 +prometheus_http_response_size_bytes_bucket{handler="/tsdb-status",le="1e+08"} 49 +prometheus_http_response_size_bytes_bucket{handler="/tsdb-status",le="1e+09"} 49 +prometheus_http_response_size_bytes_bucket{handler="/tsdb-status",le="+Inf"} 49 +prometheus_http_response_size_bytes_sum{handler="/tsdb-status"} 85848 +prometheus_http_response_size_bytes_count{handler="/tsdb-status"} 49 +# HELP prometheus_notifications_alertmanagers_discovered The number of alertmanagers discovered and active. +# TYPE prometheus_notifications_alertmanagers_discovered gauge +prometheus_notifications_alertmanagers_discovered 1 +# HELP prometheus_notifications_dropped_total Total number of alerts dropped due to errors when sending to Alertmanager. +# TYPE prometheus_notifications_dropped_total counter +prometheus_notifications_dropped_total 0 +# HELP prometheus_notifications_errors_total Total number of sent alerts affected by errors. +# TYPE prometheus_notifications_errors_total counter +prometheus_notifications_errors_total{alertmanager="http://demo.do.prometheus.io:9093/api/v2/alerts"} 0 +# HELP prometheus_notifications_latency_seconds Latency quantiles for sending alert notifications. +# TYPE prometheus_notifications_latency_seconds summary +prometheus_notifications_latency_seconds{alertmanager="http://demo.do.prometheus.io:9093/api/v2/alerts",quantile="0.5"} 0.001566044 +prometheus_notifications_latency_seconds{alertmanager="http://demo.do.prometheus.io:9093/api/v2/alerts",quantile="0.9"} 0.003927931 +prometheus_notifications_latency_seconds{alertmanager="http://demo.do.prometheus.io:9093/api/v2/alerts",quantile="0.99"} 0.013928135 +prometheus_notifications_latency_seconds_sum{alertmanager="http://demo.do.prometheus.io:9093/api/v2/alerts"} 194.15032606200046 +prometheus_notifications_latency_seconds_count{alertmanager="http://demo.do.prometheus.io:9093/api/v2/alerts"} 75180 +# HELP prometheus_notifications_queue_capacity The capacity of the alert notifications queue. +# TYPE prometheus_notifications_queue_capacity gauge +prometheus_notifications_queue_capacity 10000 +# HELP prometheus_notifications_queue_length The number of alert notifications in the queue. +# TYPE prometheus_notifications_queue_length gauge +prometheus_notifications_queue_length 0 +# HELP prometheus_notifications_sent_total Total number of alerts sent. +# TYPE prometheus_notifications_sent_total counter +prometheus_notifications_sent_total{alertmanager="http://demo.do.prometheus.io:9093/api/v2/alerts"} 141616 +# HELP prometheus_ready Whether Prometheus startup was fully completed and the server is ready for normal operation. +# TYPE prometheus_ready gauge +prometheus_ready 1 +# HELP prometheus_remote_storage_exemplars_in_total Exemplars in to remote storage, compare to exemplars out for queue managers. +# TYPE prometheus_remote_storage_exemplars_in_total counter +prometheus_remote_storage_exemplars_in_total 4.959738e+06 +# HELP prometheus_remote_storage_highest_timestamp_in_seconds Highest timestamp that has come into the remote storage via the Appender interface, in seconds since epoch. Initialized to 0 when no data has been received yet. +# TYPE prometheus_remote_storage_highest_timestamp_in_seconds gauge +prometheus_remote_storage_highest_timestamp_in_seconds 1.738949375e+09 +# HELP prometheus_remote_storage_histograms_in_total HistogramSamples in to remote storage, compare to histograms out for queue managers. +# TYPE prometheus_remote_storage_histograms_in_total counter +prometheus_remote_storage_histograms_in_total 0 +# HELP prometheus_remote_storage_samples_in_total Samples in to remote storage, compare to samples out for queue managers. +# TYPE prometheus_remote_storage_samples_in_total counter +prometheus_remote_storage_samples_in_total 6.00447296e+08 +# HELP prometheus_remote_storage_string_interner_zero_reference_releases_total The number of times release has been called for strings that are not interned. +# TYPE prometheus_remote_storage_string_interner_zero_reference_releases_total counter +prometheus_remote_storage_string_interner_zero_reference_releases_total 0 +# HELP prometheus_rule_evaluation_duration_seconds The duration for a rule to execute. +# TYPE prometheus_rule_evaluation_duration_seconds summary +prometheus_rule_evaluation_duration_seconds{quantile="0.5"} 0.000214623 +prometheus_rule_evaluation_duration_seconds{quantile="0.9"} 0.001456135 +prometheus_rule_evaluation_duration_seconds{quantile="0.99"} 0.008111814 +prometheus_rule_evaluation_duration_seconds_sum 5209.704794862625 +prometheus_rule_evaluation_duration_seconds_count 7.203456e+06 +# HELP prometheus_rule_evaluation_failures_total The total number of rule evaluation failures. +# TYPE prometheus_rule_evaluation_failures_total counter +prometheus_rule_evaluation_failures_total{rule_group="/etc/prometheus/rules/ansible_managed.rules;ansible managed alert rules"} 0 +prometheus_rule_evaluation_failures_total{rule_group="/etc/prometheus/rules/ansible_managed.yml;ansible managed alert rules"} 0 +prometheus_rule_evaluation_failures_total{rule_group="/etc/prometheus/rules/node_alerts.rules;node-exporter"} 0 +prometheus_rule_evaluation_failures_total{rule_group="/etc/prometheus/rules/node_alerts.yaml;node-exporter"} 0 +prometheus_rule_evaluation_failures_total{rule_group="/etc/prometheus/rules/node_rules.rules;node-exporter.rules"} 0 +prometheus_rule_evaluation_failures_total{rule_group="/etc/prometheus/rules/node_rules.yaml;node-exporter.rules"} 0 +prometheus_rule_evaluation_failures_total{rule_group="/etc/prometheus/rules/prometheus_alerts.rules;prometheus"} 0 +prometheus_rule_evaluation_failures_total{rule_group="/etc/prometheus/rules/prometheus_alerts.yaml;prometheus"} 0 +# HELP prometheus_rule_evaluations_total The total number of rule evaluations. +# TYPE prometheus_rule_evaluations_total counter +prometheus_rule_evaluations_total{rule_group="/etc/prometheus/rules/ansible_managed.rules;ansible managed alert rules"} 118092 +prometheus_rule_evaluations_total{rule_group="/etc/prometheus/rules/ansible_managed.yml;ansible managed alert rules"} 118090 +prometheus_rule_evaluations_total{rule_group="/etc/prometheus/rules/node_alerts.rules;node-exporter"} 1.4761e+06 +prometheus_rule_evaluations_total{rule_group="/etc/prometheus/rules/node_alerts.yaml;node-exporter"} 1.476125e+06 +prometheus_rule_evaluations_total{rule_group="/etc/prometheus/rules/node_rules.rules;node-exporter.rules"} 649495 +prometheus_rule_evaluations_total{rule_group="/etc/prometheus/rules/node_rules.yaml;node-exporter.rules"} 649484 +prometheus_rule_evaluations_total{rule_group="/etc/prometheus/rules/prometheus_alerts.rules;prometheus"} 1.358035e+06 +prometheus_rule_evaluations_total{rule_group="/etc/prometheus/rules/prometheus_alerts.yaml;prometheus"} 1.358035e+06 +# HELP prometheus_rule_group_duration_seconds The duration of rule group evaluations. +# TYPE prometheus_rule_group_duration_seconds summary +prometheus_rule_group_duration_seconds{quantile="0.01"} 0.000735928 +prometheus_rule_group_duration_seconds{quantile="0.05"} 0.000818857 +prometheus_rule_group_duration_seconds{quantile="0.5"} 0.004852081 +prometheus_rule_group_duration_seconds{quantile="0.9"} 0.022897759 +prometheus_rule_group_duration_seconds{quantile="0.99"} 0.069327797 +prometheus_rule_group_duration_seconds_sum 5335.451440133042 +prometheus_rule_group_duration_seconds_count 472359 +# HELP prometheus_rule_group_interval_seconds The interval of a rule group. +# TYPE prometheus_rule_group_interval_seconds gauge +prometheus_rule_group_interval_seconds{rule_group="/etc/prometheus/rules/ansible_managed.rules;ansible managed alert rules"} 15 +prometheus_rule_group_interval_seconds{rule_group="/etc/prometheus/rules/ansible_managed.yml;ansible managed alert rules"} 15 +prometheus_rule_group_interval_seconds{rule_group="/etc/prometheus/rules/node_alerts.rules;node-exporter"} 15 +prometheus_rule_group_interval_seconds{rule_group="/etc/prometheus/rules/node_alerts.yaml;node-exporter"} 15 +prometheus_rule_group_interval_seconds{rule_group="/etc/prometheus/rules/node_rules.rules;node-exporter.rules"} 15 +prometheus_rule_group_interval_seconds{rule_group="/etc/prometheus/rules/node_rules.yaml;node-exporter.rules"} 15 +prometheus_rule_group_interval_seconds{rule_group="/etc/prometheus/rules/prometheus_alerts.rules;prometheus"} 15 +prometheus_rule_group_interval_seconds{rule_group="/etc/prometheus/rules/prometheus_alerts.yaml;prometheus"} 15 +# HELP prometheus_rule_group_iterations_missed_total The total number of rule group evaluations missed due to slow rule group evaluation. +# TYPE prometheus_rule_group_iterations_missed_total counter +prometheus_rule_group_iterations_missed_total{rule_group="/etc/prometheus/rules/ansible_managed.rules;ansible managed alert rules"} 0 +prometheus_rule_group_iterations_missed_total{rule_group="/etc/prometheus/rules/ansible_managed.yml;ansible managed alert rules"} 0 +prometheus_rule_group_iterations_missed_total{rule_group="/etc/prometheus/rules/node_alerts.rules;node-exporter"} 1 +prometheus_rule_group_iterations_missed_total{rule_group="/etc/prometheus/rules/node_alerts.yaml;node-exporter"} 0 +prometheus_rule_group_iterations_missed_total{rule_group="/etc/prometheus/rules/node_rules.rules;node-exporter.rules"} 0 +prometheus_rule_group_iterations_missed_total{rule_group="/etc/prometheus/rules/node_rules.yaml;node-exporter.rules"} 1 +prometheus_rule_group_iterations_missed_total{rule_group="/etc/prometheus/rules/prometheus_alerts.rules;prometheus"} 0 +prometheus_rule_group_iterations_missed_total{rule_group="/etc/prometheus/rules/prometheus_alerts.yaml;prometheus"} 0 +# HELP prometheus_rule_group_iterations_total The total number of scheduled rule group evaluations, whether executed or missed. +# TYPE prometheus_rule_group_iterations_total counter +prometheus_rule_group_iterations_total{rule_group="/etc/prometheus/rules/ansible_managed.rules;ansible managed alert rules"} 59046 +prometheus_rule_group_iterations_total{rule_group="/etc/prometheus/rules/ansible_managed.yml;ansible managed alert rules"} 59045 +prometheus_rule_group_iterations_total{rule_group="/etc/prometheus/rules/node_alerts.rules;node-exporter"} 59045 +prometheus_rule_group_iterations_total{rule_group="/etc/prometheus/rules/node_alerts.yaml;node-exporter"} 59045 +prometheus_rule_group_iterations_total{rule_group="/etc/prometheus/rules/node_rules.rules;node-exporter.rules"} 59045 +prometheus_rule_group_iterations_total{rule_group="/etc/prometheus/rules/node_rules.yaml;node-exporter.rules"} 59045 +prometheus_rule_group_iterations_total{rule_group="/etc/prometheus/rules/prometheus_alerts.rules;prometheus"} 59045 +prometheus_rule_group_iterations_total{rule_group="/etc/prometheus/rules/prometheus_alerts.yaml;prometheus"} 59045 +# HELP prometheus_rule_group_last_duration_seconds The duration of the last rule group evaluation. +# TYPE prometheus_rule_group_last_duration_seconds gauge +prometheus_rule_group_last_duration_seconds{rule_group="/etc/prometheus/rules/ansible_managed.rules;ansible managed alert rules"} 0.000754015 +prometheus_rule_group_last_duration_seconds{rule_group="/etc/prometheus/rules/ansible_managed.yml;ansible managed alert rules"} 0.001104624 +prometheus_rule_group_last_duration_seconds{rule_group="/etc/prometheus/rules/node_alerts.rules;node-exporter"} 0.022040842 +prometheus_rule_group_last_duration_seconds{rule_group="/etc/prometheus/rules/node_alerts.yaml;node-exporter"} 0.048928087 +prometheus_rule_group_last_duration_seconds{rule_group="/etc/prometheus/rules/node_rules.rules;node-exporter.rules"} 0.002766703 +prometheus_rule_group_last_duration_seconds{rule_group="/etc/prometheus/rules/node_rules.yaml;node-exporter.rules"} 0.002218466 +prometheus_rule_group_last_duration_seconds{rule_group="/etc/prometheus/rules/prometheus_alerts.rules;prometheus"} 0.010245447 +prometheus_rule_group_last_duration_seconds{rule_group="/etc/prometheus/rules/prometheus_alerts.yaml;prometheus"} 0.009232393 +# HELP prometheus_rule_group_last_evaluation_samples The number of samples returned during the last rule group evaluation. +# TYPE prometheus_rule_group_last_evaluation_samples gauge +prometheus_rule_group_last_evaluation_samples{rule_group="/etc/prometheus/rules/ansible_managed.rules;ansible managed alert rules"} 2 +prometheus_rule_group_last_evaluation_samples{rule_group="/etc/prometheus/rules/ansible_managed.yml;ansible managed alert rules"} 2 +prometheus_rule_group_last_evaluation_samples{rule_group="/etc/prometheus/rules/node_alerts.rules;node-exporter"} 10 +prometheus_rule_group_last_evaluation_samples{rule_group="/etc/prometheus/rules/node_alerts.yaml;node-exporter"} 10 +prometheus_rule_group_last_evaluation_samples{rule_group="/etc/prometheus/rules/node_rules.rules;node-exporter.rules"} 11 +prometheus_rule_group_last_evaluation_samples{rule_group="/etc/prometheus/rules/node_rules.yaml;node-exporter.rules"} 11 +prometheus_rule_group_last_evaluation_samples{rule_group="/etc/prometheus/rules/prometheus_alerts.rules;prometheus"} 0 +prometheus_rule_group_last_evaluation_samples{rule_group="/etc/prometheus/rules/prometheus_alerts.yaml;prometheus"} 0 +# HELP prometheus_rule_group_last_evaluation_timestamp_seconds The timestamp of the last rule group evaluation in seconds. +# TYPE prometheus_rule_group_last_evaluation_timestamp_seconds gauge +prometheus_rule_group_last_evaluation_timestamp_seconds{rule_group="/etc/prometheus/rules/ansible_managed.rules;ansible managed alert rules"} 1.738949373753179e+09 +prometheus_rule_group_last_evaluation_timestamp_seconds{rule_group="/etc/prometheus/rules/ansible_managed.yml;ansible managed alert rules"} 1.738949366900781e+09 +prometheus_rule_group_last_evaluation_timestamp_seconds{rule_group="/etc/prometheus/rules/node_alerts.rules;node-exporter"} 1.7389493632087085e+09 +prometheus_rule_group_last_evaluation_timestamp_seconds{rule_group="/etc/prometheus/rules/node_alerts.yaml;node-exporter"} 1.7389493666743164e+09 +prometheus_rule_group_last_evaluation_timestamp_seconds{rule_group="/etc/prometheus/rules/node_rules.rules;node-exporter.rules"} 1.7389493675395968e+09 +prometheus_rule_group_last_evaluation_timestamp_seconds{rule_group="/etc/prometheus/rules/node_rules.yaml;node-exporter.rules"} 1.7389493623381927e+09 +prometheus_rule_group_last_evaluation_timestamp_seconds{rule_group="/etc/prometheus/rules/prometheus_alerts.rules;prometheus"} 1.738949366856423e+09 +prometheus_rule_group_last_evaluation_timestamp_seconds{rule_group="/etc/prometheus/rules/prometheus_alerts.yaml;prometheus"} 1.738949369917499e+09 +# HELP prometheus_rule_group_last_restore_duration_seconds The duration of the last alert rules alerts restoration using the `ALERTS_FOR_STATE` series. +# TYPE prometheus_rule_group_last_restore_duration_seconds gauge +prometheus_rule_group_last_restore_duration_seconds{rule_group="/etc/prometheus/rules/ansible_managed.rules;ansible managed alert rules"} 0.002032625 +prometheus_rule_group_last_restore_duration_seconds{rule_group="/etc/prometheus/rules/ansible_managed.yml;ansible managed alert rules"} 9.1853e-05 +prometheus_rule_group_last_restore_duration_seconds{rule_group="/etc/prometheus/rules/node_alerts.rules;node-exporter"} 0.000166088 +prometheus_rule_group_last_restore_duration_seconds{rule_group="/etc/prometheus/rules/node_alerts.yaml;node-exporter"} 0.000108127 +prometheus_rule_group_last_restore_duration_seconds{rule_group="/etc/prometheus/rules/node_rules.rules;node-exporter.rules"} 6.408e-06 +prometheus_rule_group_last_restore_duration_seconds{rule_group="/etc/prometheus/rules/node_rules.yaml;node-exporter.rules"} 2.621e-06 +prometheus_rule_group_last_restore_duration_seconds{rule_group="/etc/prometheus/rules/prometheus_alerts.rules;prometheus"} 8.4979e-05 +prometheus_rule_group_last_restore_duration_seconds{rule_group="/etc/prometheus/rules/prometheus_alerts.yaml;prometheus"} 0.000104444 +# HELP prometheus_rule_group_last_rule_duration_sum_seconds The sum of time in seconds it took to evaluate each rule in the group regardless of concurrency. This should be higher than the group duration if rules are evaluated concurrently. +# TYPE prometheus_rule_group_last_rule_duration_sum_seconds gauge +prometheus_rule_group_last_rule_duration_sum_seconds{rule_group="/etc/prometheus/rules/ansible_managed.rules;ansible managed alert rules"} 0.000685176 +prometheus_rule_group_last_rule_duration_sum_seconds{rule_group="/etc/prometheus/rules/ansible_managed.yml;ansible managed alert rules"} 0.001035099 +prometheus_rule_group_last_rule_duration_sum_seconds{rule_group="/etc/prometheus/rules/node_alerts.rules;node-exporter"} 0.021769662 +prometheus_rule_group_last_rule_duration_sum_seconds{rule_group="/etc/prometheus/rules/node_alerts.yaml;node-exporter"} 0.042017156 +prometheus_rule_group_last_rule_duration_sum_seconds{rule_group="/etc/prometheus/rules/node_rules.rules;node-exporter.rules"} 0.00260535 +prometheus_rule_group_last_rule_duration_sum_seconds{rule_group="/etc/prometheus/rules/node_rules.yaml;node-exporter.rules"} 0.002069356 +prometheus_rule_group_last_rule_duration_sum_seconds{rule_group="/etc/prometheus/rules/prometheus_alerts.rules;prometheus"} 0.009905481 +prometheus_rule_group_last_rule_duration_sum_seconds{rule_group="/etc/prometheus/rules/prometheus_alerts.yaml;prometheus"} 0.009056893 +# HELP prometheus_rule_group_rules The number of rules. +# TYPE prometheus_rule_group_rules gauge +prometheus_rule_group_rules{rule_group="/etc/prometheus/rules/ansible_managed.rules;ansible managed alert rules"} 2 +prometheus_rule_group_rules{rule_group="/etc/prometheus/rules/ansible_managed.yml;ansible managed alert rules"} 2 +prometheus_rule_group_rules{rule_group="/etc/prometheus/rules/node_alerts.rules;node-exporter"} 25 +prometheus_rule_group_rules{rule_group="/etc/prometheus/rules/node_alerts.yaml;node-exporter"} 25 +prometheus_rule_group_rules{rule_group="/etc/prometheus/rules/node_rules.rules;node-exporter.rules"} 11 +prometheus_rule_group_rules{rule_group="/etc/prometheus/rules/node_rules.yaml;node-exporter.rules"} 11 +prometheus_rule_group_rules{rule_group="/etc/prometheus/rules/prometheus_alerts.rules;prometheus"} 23 +prometheus_rule_group_rules{rule_group="/etc/prometheus/rules/prometheus_alerts.yaml;prometheus"} 23 +# HELP prometheus_sd_azure_cache_hit_total Number of cache hit during refresh. +# TYPE prometheus_sd_azure_cache_hit_total counter +prometheus_sd_azure_cache_hit_total 0 +# HELP prometheus_sd_azure_failures_total Number of Azure service discovery refresh failures. +# TYPE prometheus_sd_azure_failures_total counter +prometheus_sd_azure_failures_total 0 +# HELP prometheus_sd_consul_rpc_duration_seconds The duration of a Consul RPC call in seconds. +# TYPE prometheus_sd_consul_rpc_duration_seconds summary +prometheus_sd_consul_rpc_duration_seconds{call="service",endpoint="catalog",quantile="0.5"} NaN +prometheus_sd_consul_rpc_duration_seconds{call="service",endpoint="catalog",quantile="0.9"} NaN +prometheus_sd_consul_rpc_duration_seconds{call="service",endpoint="catalog",quantile="0.99"} NaN +prometheus_sd_consul_rpc_duration_seconds_sum{call="service",endpoint="catalog"} 0 +prometheus_sd_consul_rpc_duration_seconds_count{call="service",endpoint="catalog"} 0 +prometheus_sd_consul_rpc_duration_seconds{call="services",endpoint="catalog",quantile="0.5"} NaN +prometheus_sd_consul_rpc_duration_seconds{call="services",endpoint="catalog",quantile="0.9"} NaN +prometheus_sd_consul_rpc_duration_seconds{call="services",endpoint="catalog",quantile="0.99"} NaN +prometheus_sd_consul_rpc_duration_seconds_sum{call="services",endpoint="catalog"} 0 +prometheus_sd_consul_rpc_duration_seconds_count{call="services",endpoint="catalog"} 0 +# HELP prometheus_sd_consul_rpc_failures_total The number of Consul RPC call failures. +# TYPE prometheus_sd_consul_rpc_failures_total counter +prometheus_sd_consul_rpc_failures_total 0 +# HELP prometheus_sd_discovered_targets Current number of discovered targets. +# TYPE prometheus_sd_discovered_targets gauge +prometheus_sd_discovered_targets{config="alertmanager",name="scrape"} 1 +prometheus_sd_discovered_targets{config="blackbox",name="scrape"} 1 +prometheus_sd_discovered_targets{config="caddy",name="scrape"} 1 +prometheus_sd_discovered_targets{config="cadvisor",name="scrape"} 1 +prometheus_sd_discovered_targets{config="config-0",name="notify"} 1 +prometheus_sd_discovered_targets{config="grafana",name="scrape"} 1 +prometheus_sd_discovered_targets{config="node",name="scrape"} 1 +prometheus_sd_discovered_targets{config="prometheus",name="scrape"} 1 +prometheus_sd_discovered_targets{config="random",name="scrape"} 4 +# HELP prometheus_sd_dns_lookup_failures_total The number of DNS-SD lookup failures. +# TYPE prometheus_sd_dns_lookup_failures_total counter +prometheus_sd_dns_lookup_failures_total 0 +# HELP prometheus_sd_dns_lookups_total The number of DNS-SD lookups. +# TYPE prometheus_sd_dns_lookups_total counter +prometheus_sd_dns_lookups_total 0 +# HELP prometheus_sd_failed_configs Current number of service discovery configurations that failed to load. +# TYPE prometheus_sd_failed_configs gauge +prometheus_sd_failed_configs{name="notify"} 0 +prometheus_sd_failed_configs{name="scrape"} 0 +# HELP prometheus_sd_file_mtime_seconds Timestamp (mtime) of files read by FileSD. Timestamp is set at read time. +# TYPE prometheus_sd_file_mtime_seconds gauge +prometheus_sd_file_mtime_seconds{filename="/etc/prometheus/file_sd/alertmanager.yml"} 1.701295557e+09 +prometheus_sd_file_mtime_seconds{filename="/etc/prometheus/file_sd/cadvisor.yml"} 1.705433682e+09 +prometheus_sd_file_mtime_seconds{filename="/etc/prometheus/file_sd/node.yml"} 1.701295553e+09 +prometheus_sd_file_mtime_seconds{filename="/etc/prometheus/file_sd/random.yml"} 1.579263729e+09 +# HELP prometheus_sd_file_read_errors_total The number of File-SD read errors. +# TYPE prometheus_sd_file_read_errors_total counter +prometheus_sd_file_read_errors_total 0 +# HELP prometheus_sd_file_scan_duration_seconds The duration of the File-SD scan in seconds. +# TYPE prometheus_sd_file_scan_duration_seconds summary +prometheus_sd_file_scan_duration_seconds{quantile="0.5"} 9.5384e-05 +prometheus_sd_file_scan_duration_seconds{quantile="0.9"} 0.000275679 +prometheus_sd_file_scan_duration_seconds{quantile="0.99"} 0.000275679 +prometheus_sd_file_scan_duration_seconds_sum 4.751347856999997 +prometheus_sd_file_scan_duration_seconds_count 11812 +# HELP prometheus_sd_file_watcher_errors_total The number of File-SD errors caused by filesystem watch failures. +# TYPE prometheus_sd_file_watcher_errors_total counter +prometheus_sd_file_watcher_errors_total 0 +# HELP prometheus_sd_http_failures_total Number of HTTP service discovery refresh failures. +# TYPE prometheus_sd_http_failures_total counter +prometheus_sd_http_failures_total 0 +# HELP prometheus_sd_kubernetes_events_total The number of Kubernetes events handled. +# TYPE prometheus_sd_kubernetes_events_total counter +prometheus_sd_kubernetes_events_total{event="add",role="endpoints"} 0 +prometheus_sd_kubernetes_events_total{event="add",role="endpointslice"} 0 +prometheus_sd_kubernetes_events_total{event="add",role="ingress"} 0 +prometheus_sd_kubernetes_events_total{event="add",role="node"} 0 +prometheus_sd_kubernetes_events_total{event="add",role="pod"} 0 +prometheus_sd_kubernetes_events_total{event="add",role="service"} 0 +prometheus_sd_kubernetes_events_total{event="delete",role="endpoints"} 0 +prometheus_sd_kubernetes_events_total{event="delete",role="endpointslice"} 0 +prometheus_sd_kubernetes_events_total{event="delete",role="ingress"} 0 +prometheus_sd_kubernetes_events_total{event="delete",role="node"} 0 +prometheus_sd_kubernetes_events_total{event="delete",role="pod"} 0 +prometheus_sd_kubernetes_events_total{event="delete",role="service"} 0 +prometheus_sd_kubernetes_events_total{event="update",role="endpoints"} 0 +prometheus_sd_kubernetes_events_total{event="update",role="endpointslice"} 0 +prometheus_sd_kubernetes_events_total{event="update",role="ingress"} 0 +prometheus_sd_kubernetes_events_total{event="update",role="node"} 0 +prometheus_sd_kubernetes_events_total{event="update",role="pod"} 0 +prometheus_sd_kubernetes_events_total{event="update",role="service"} 0 +# HELP prometheus_sd_kubernetes_failures_total The number of failed WATCH/LIST requests. +# TYPE prometheus_sd_kubernetes_failures_total counter +prometheus_sd_kubernetes_failures_total 0 +# HELP prometheus_sd_kuma_fetch_duration_seconds The duration of a Kuma MADS fetch call. +# TYPE prometheus_sd_kuma_fetch_duration_seconds summary +prometheus_sd_kuma_fetch_duration_seconds{quantile="0.5"} NaN +prometheus_sd_kuma_fetch_duration_seconds{quantile="0.9"} NaN +prometheus_sd_kuma_fetch_duration_seconds{quantile="0.99"} NaN +prometheus_sd_kuma_fetch_duration_seconds_sum 0 +prometheus_sd_kuma_fetch_duration_seconds_count 0 +# HELP prometheus_sd_kuma_fetch_failures_total The number of Kuma MADS fetch call failures. +# TYPE prometheus_sd_kuma_fetch_failures_total counter +prometheus_sd_kuma_fetch_failures_total 0 +# HELP prometheus_sd_kuma_fetch_skipped_updates_total The number of Kuma MADS fetch calls that result in no updates to the targets. +# TYPE prometheus_sd_kuma_fetch_skipped_updates_total counter +prometheus_sd_kuma_fetch_skipped_updates_total 0 +# HELP prometheus_sd_linode_failures_total Number of Linode service discovery refresh failures. +# TYPE prometheus_sd_linode_failures_total counter +prometheus_sd_linode_failures_total 0 +# HELP prometheus_sd_nomad_failures_total Number of nomad service discovery refresh failures. +# TYPE prometheus_sd_nomad_failures_total counter +prometheus_sd_nomad_failures_total 0 +# HELP prometheus_sd_received_updates_total Total number of update events received from the SD providers. +# TYPE prometheus_sd_received_updates_total counter +prometheus_sd_received_updates_total{name="notify"} 2 +prometheus_sd_received_updates_total{name="scrape"} 11820 +# HELP prometheus_sd_updates_delayed_total Total number of update events that couldn't be sent immediately. +# TYPE prometheus_sd_updates_delayed_total counter +prometheus_sd_updates_delayed_total{name="notify"} 0 +prometheus_sd_updates_delayed_total{name="scrape"} 0 +# HELP prometheus_sd_updates_total Total number of update events sent to the SD consumers. +# TYPE prometheus_sd_updates_total counter +prometheus_sd_updates_total{name="notify"} 1 +prometheus_sd_updates_total{name="scrape"} 2953 +# HELP prometheus_target_interval_length_seconds Actual intervals between scrapes. +# TYPE prometheus_target_interval_length_seconds summary +prometheus_target_interval_length_seconds{interval="15s",quantile="0.01"} 14.982591232 +prometheus_target_interval_length_seconds{interval="15s",quantile="0.05"} 14.997567414 +prometheus_target_interval_length_seconds{interval="15s",quantile="0.5"} 14.999977915 +prometheus_target_interval_length_seconds{interval="15s",quantile="0.9"} 15.000793403 +prometheus_target_interval_length_seconds{interval="15s",quantile="0.99"} 15.017607167 +prometheus_target_interval_length_seconds_sum{interval="15s"} 9.742237453453667e+06 +prometheus_target_interval_length_seconds_count{interval="15s"} 649482 +# HELP prometheus_target_metadata_cache_bytes The number of bytes that are currently used for storing metric metadata in the cache +# TYPE prometheus_target_metadata_cache_bytes gauge +prometheus_target_metadata_cache_bytes{scrape_job="alertmanager"} 6817 +prometheus_target_metadata_cache_bytes{scrape_job="blackbox"} 661 +prometheus_target_metadata_cache_bytes{scrape_job="caddy"} 2365 +prometheus_target_metadata_cache_bytes{scrape_job="cadvisor"} 4547 +prometheus_target_metadata_cache_bytes{scrape_job="grafana"} 37608 +prometheus_target_metadata_cache_bytes{scrape_job="node"} 13900 +prometheus_target_metadata_cache_bytes{scrape_job="prometheus"} 20265 +prometheus_target_metadata_cache_bytes{scrape_job="random"} 792 +# HELP prometheus_target_metadata_cache_entries Total number of metric metadata entries in the cache +# TYPE prometheus_target_metadata_cache_entries gauge +prometheus_target_metadata_cache_entries{scrape_job="alertmanager"} 86 +prometheus_target_metadata_cache_entries{scrape_job="blackbox"} 13 +prometheus_target_metadata_cache_entries{scrape_job="caddy"} 47 +prometheus_target_metadata_cache_entries{scrape_job="cadvisor"} 93 +prometheus_target_metadata_cache_entries{scrape_job="grafana"} 373 +prometheus_target_metadata_cache_entries{scrape_job="node"} 293 +prometheus_target_metadata_cache_entries{scrape_job="prometheus"} 237 +prometheus_target_metadata_cache_entries{scrape_job="random"} 16 +# HELP prometheus_target_scrape_pool_exceeded_label_limits_total Total number of times scrape pools hit the label limits, during sync or config reload. +# TYPE prometheus_target_scrape_pool_exceeded_label_limits_total counter +prometheus_target_scrape_pool_exceeded_label_limits_total 0 +# HELP prometheus_target_scrape_pool_exceeded_target_limit_total Total number of times scrape pools hit the target limit, during sync or config reload. +# TYPE prometheus_target_scrape_pool_exceeded_target_limit_total counter +prometheus_target_scrape_pool_exceeded_target_limit_total 0 +# HELP prometheus_target_scrape_pool_reloads_failed_total Total number of failed scrape pool reloads. +# TYPE prometheus_target_scrape_pool_reloads_failed_total counter +prometheus_target_scrape_pool_reloads_failed_total 0 +# HELP prometheus_target_scrape_pool_reloads_total Total number of scrape pool reloads. +# TYPE prometheus_target_scrape_pool_reloads_total counter +prometheus_target_scrape_pool_reloads_total 0 +# HELP prometheus_target_scrape_pool_symboltable_items Current number of symbols in table for this scrape pool. +# TYPE prometheus_target_scrape_pool_symboltable_items gauge +prometheus_target_scrape_pool_symboltable_items{scrape_job="alertmanager"} 0 +prometheus_target_scrape_pool_symboltable_items{scrape_job="blackbox"} 0 +prometheus_target_scrape_pool_symboltable_items{scrape_job="caddy"} 0 +prometheus_target_scrape_pool_symboltable_items{scrape_job="cadvisor"} 0 +prometheus_target_scrape_pool_symboltable_items{scrape_job="grafana"} 0 +prometheus_target_scrape_pool_symboltable_items{scrape_job="node"} 0 +prometheus_target_scrape_pool_symboltable_items{scrape_job="prometheus"} 0 +prometheus_target_scrape_pool_symboltable_items{scrape_job="random"} 0 +# HELP prometheus_target_scrape_pool_sync_total Total number of syncs that were executed on a scrape pool. +# TYPE prometheus_target_scrape_pool_sync_total counter +prometheus_target_scrape_pool_sync_total{scrape_job="alertmanager"} 2953 +prometheus_target_scrape_pool_sync_total{scrape_job="blackbox"} 2953 +prometheus_target_scrape_pool_sync_total{scrape_job="caddy"} 2953 +prometheus_target_scrape_pool_sync_total{scrape_job="cadvisor"} 2953 +prometheus_target_scrape_pool_sync_total{scrape_job="grafana"} 2953 +prometheus_target_scrape_pool_sync_total{scrape_job="node"} 2953 +prometheus_target_scrape_pool_sync_total{scrape_job="prometheus"} 2953 +prometheus_target_scrape_pool_sync_total{scrape_job="random"} 2953 +# HELP prometheus_target_scrape_pool_target_limit Maximum number of targets allowed in this scrape pool. +# TYPE prometheus_target_scrape_pool_target_limit gauge +prometheus_target_scrape_pool_target_limit{scrape_job="alertmanager"} 0 +prometheus_target_scrape_pool_target_limit{scrape_job="blackbox"} 0 +prometheus_target_scrape_pool_target_limit{scrape_job="caddy"} 0 +prometheus_target_scrape_pool_target_limit{scrape_job="cadvisor"} 0 +prometheus_target_scrape_pool_target_limit{scrape_job="grafana"} 0 +prometheus_target_scrape_pool_target_limit{scrape_job="node"} 0 +prometheus_target_scrape_pool_target_limit{scrape_job="prometheus"} 0 +prometheus_target_scrape_pool_target_limit{scrape_job="random"} 0 +# HELP prometheus_target_scrape_pool_targets Current number of targets in this scrape pool. +# TYPE prometheus_target_scrape_pool_targets gauge +prometheus_target_scrape_pool_targets{scrape_job="alertmanager"} 1 +prometheus_target_scrape_pool_targets{scrape_job="blackbox"} 1 +prometheus_target_scrape_pool_targets{scrape_job="caddy"} 1 +prometheus_target_scrape_pool_targets{scrape_job="cadvisor"} 1 +prometheus_target_scrape_pool_targets{scrape_job="grafana"} 1 +prometheus_target_scrape_pool_targets{scrape_job="node"} 1 +prometheus_target_scrape_pool_targets{scrape_job="prometheus"} 1 +prometheus_target_scrape_pool_targets{scrape_job="random"} 4 +# HELP prometheus_target_scrape_pools_failed_total Total number of scrape pool creations that failed. +# TYPE prometheus_target_scrape_pools_failed_total counter +prometheus_target_scrape_pools_failed_total 0 +# HELP prometheus_target_scrape_pools_total Total number of scrape pool creation attempts. +# TYPE prometheus_target_scrape_pools_total counter +prometheus_target_scrape_pools_total 8 +# HELP prometheus_target_scrapes_cache_flush_forced_total How many times a scrape cache was flushed due to getting big while scrapes are failing. +# TYPE prometheus_target_scrapes_cache_flush_forced_total counter +prometheus_target_scrapes_cache_flush_forced_total 0 +# HELP prometheus_target_scrapes_exceeded_body_size_limit_total Total number of scrapes that hit the body size limit +# TYPE prometheus_target_scrapes_exceeded_body_size_limit_total counter +prometheus_target_scrapes_exceeded_body_size_limit_total 0 +# HELP prometheus_target_scrapes_exceeded_native_histogram_bucket_limit_total Total number of scrapes that hit the native histogram bucket limit and were rejected. +# TYPE prometheus_target_scrapes_exceeded_native_histogram_bucket_limit_total counter +prometheus_target_scrapes_exceeded_native_histogram_bucket_limit_total 0 +# HELP prometheus_target_scrapes_exceeded_sample_limit_total Total number of scrapes that hit the sample limit and were rejected. +# TYPE prometheus_target_scrapes_exceeded_sample_limit_total counter +prometheus_target_scrapes_exceeded_sample_limit_total 0 +# HELP prometheus_target_scrapes_exemplar_out_of_order_total Total number of exemplar rejected due to not being out of the expected order. +# TYPE prometheus_target_scrapes_exemplar_out_of_order_total counter +prometheus_target_scrapes_exemplar_out_of_order_total 0 +# HELP prometheus_target_scrapes_sample_duplicate_timestamp_total Total number of samples rejected due to duplicate timestamps but different values. +# TYPE prometheus_target_scrapes_sample_duplicate_timestamp_total counter +prometheus_target_scrapes_sample_duplicate_timestamp_total 0 +# HELP prometheus_target_scrapes_sample_out_of_bounds_total Total number of samples rejected due to timestamp falling outside of the time bounds. +# TYPE prometheus_target_scrapes_sample_out_of_bounds_total counter +prometheus_target_scrapes_sample_out_of_bounds_total 0 +# HELP prometheus_target_scrapes_sample_out_of_order_total Total number of samples rejected due to not being out of the expected order. +# TYPE prometheus_target_scrapes_sample_out_of_order_total counter +prometheus_target_scrapes_sample_out_of_order_total 455 +# HELP prometheus_target_sync_failed_total Total number of target sync failures. +# TYPE prometheus_target_sync_failed_total counter +prometheus_target_sync_failed_total{scrape_job="alertmanager"} 0 +prometheus_target_sync_failed_total{scrape_job="blackbox"} 0 +prometheus_target_sync_failed_total{scrape_job="caddy"} 0 +prometheus_target_sync_failed_total{scrape_job="cadvisor"} 0 +prometheus_target_sync_failed_total{scrape_job="grafana"} 0 +prometheus_target_sync_failed_total{scrape_job="node"} 0 +prometheus_target_sync_failed_total{scrape_job="prometheus"} 0 +prometheus_target_sync_failed_total{scrape_job="random"} 0 +# HELP prometheus_target_sync_length_seconds Actual interval to sync the scrape pool. +# TYPE prometheus_target_sync_length_seconds summary +prometheus_target_sync_length_seconds{scrape_job="alertmanager",quantile="0.01"} 2.0522e-05 +prometheus_target_sync_length_seconds{scrape_job="alertmanager",quantile="0.05"} 2.0522e-05 +prometheus_target_sync_length_seconds{scrape_job="alertmanager",quantile="0.5"} 2.0522e-05 +prometheus_target_sync_length_seconds{scrape_job="alertmanager",quantile="0.9"} 0.000141485 +prometheus_target_sync_length_seconds{scrape_job="alertmanager",quantile="0.99"} 0.000141485 +prometheus_target_sync_length_seconds_sum{scrape_job="alertmanager"} 0.13103036 +prometheus_target_sync_length_seconds_count{scrape_job="alertmanager"} 2953 +prometheus_target_sync_length_seconds{scrape_job="blackbox",quantile="0.01"} 3.9252e-05 +prometheus_target_sync_length_seconds{scrape_job="blackbox",quantile="0.05"} 3.9252e-05 +prometheus_target_sync_length_seconds{scrape_job="blackbox",quantile="0.5"} 3.9252e-05 +prometheus_target_sync_length_seconds{scrape_job="blackbox",quantile="0.9"} 6.2134e-05 +prometheus_target_sync_length_seconds{scrape_job="blackbox",quantile="0.99"} 6.2134e-05 +prometheus_target_sync_length_seconds_sum{scrape_job="blackbox"} 0.6044201539999996 +prometheus_target_sync_length_seconds_count{scrape_job="blackbox"} 2953 +prometheus_target_sync_length_seconds{scrape_job="caddy",quantile="0.01"} 1.3759e-05 +prometheus_target_sync_length_seconds{scrape_job="caddy",quantile="0.05"} 1.3759e-05 +prometheus_target_sync_length_seconds{scrape_job="caddy",quantile="0.5"} 1.3759e-05 +prometheus_target_sync_length_seconds{scrape_job="caddy",quantile="0.9"} 7.8256e-05 +prometheus_target_sync_length_seconds{scrape_job="caddy",quantile="0.99"} 7.8256e-05 +prometheus_target_sync_length_seconds_sum{scrape_job="caddy"} 0.10369844599999971 +prometheus_target_sync_length_seconds_count{scrape_job="caddy"} 2953 +prometheus_target_sync_length_seconds{scrape_job="cadvisor",quantile="0.01"} 2.0452e-05 +prometheus_target_sync_length_seconds{scrape_job="cadvisor",quantile="0.05"} 2.0452e-05 +prometheus_target_sync_length_seconds{scrape_job="cadvisor",quantile="0.5"} 2.0452e-05 +prometheus_target_sync_length_seconds{scrape_job="cadvisor",quantile="0.9"} 4.2337e-05 +prometheus_target_sync_length_seconds{scrape_job="cadvisor",quantile="0.99"} 4.2337e-05 +prometheus_target_sync_length_seconds_sum{scrape_job="cadvisor"} 0.10489659999999998 +prometheus_target_sync_length_seconds_count{scrape_job="cadvisor"} 2953 +prometheus_target_sync_length_seconds{scrape_job="grafana",quantile="0.01"} 1.4995e-05 +prometheus_target_sync_length_seconds{scrape_job="grafana",quantile="0.05"} 1.4995e-05 +prometheus_target_sync_length_seconds{scrape_job="grafana",quantile="0.5"} 1.4995e-05 +prometheus_target_sync_length_seconds{scrape_job="grafana",quantile="0.9"} 1.7284e-05 +prometheus_target_sync_length_seconds{scrape_job="grafana",quantile="0.99"} 1.7284e-05 +prometheus_target_sync_length_seconds_sum{scrape_job="grafana"} 0.09031192700000017 +prometheus_target_sync_length_seconds_count{scrape_job="grafana"} 2953 +prometheus_target_sync_length_seconds{scrape_job="node",quantile="0.01"} 4.1607e-05 +prometheus_target_sync_length_seconds{scrape_job="node",quantile="0.05"} 4.1607e-05 +prometheus_target_sync_length_seconds{scrape_job="node",quantile="0.5"} 4.1607e-05 +prometheus_target_sync_length_seconds{scrape_job="node",quantile="0.9"} 7.416e-05 +prometheus_target_sync_length_seconds{scrape_job="node",quantile="0.99"} 7.416e-05 +prometheus_target_sync_length_seconds_sum{scrape_job="node"} 0.11539821299999993 +prometheus_target_sync_length_seconds_count{scrape_job="node"} 2953 +prometheus_target_sync_length_seconds{scrape_job="prometheus",quantile="0.01"} 1.5564e-05 +prometheus_target_sync_length_seconds{scrape_job="prometheus",quantile="0.05"} 1.5564e-05 +prometheus_target_sync_length_seconds{scrape_job="prometheus",quantile="0.5"} 1.5564e-05 +prometheus_target_sync_length_seconds{scrape_job="prometheus",quantile="0.9"} 1.961e-05 +prometheus_target_sync_length_seconds{scrape_job="prometheus",quantile="0.99"} 1.961e-05 +prometheus_target_sync_length_seconds_sum{scrape_job="prometheus"} 0.10655758600000016 +prometheus_target_sync_length_seconds_count{scrape_job="prometheus"} 2953 +prometheus_target_sync_length_seconds{scrape_job="random",quantile="0.01"} 4.1299e-05 +prometheus_target_sync_length_seconds{scrape_job="random",quantile="0.05"} 4.1299e-05 +prometheus_target_sync_length_seconds{scrape_job="random",quantile="0.5"} 4.1299e-05 +prometheus_target_sync_length_seconds{scrape_job="random",quantile="0.9"} 4.8586e-05 +prometheus_target_sync_length_seconds{scrape_job="random",quantile="0.99"} 4.8586e-05 +prometheus_target_sync_length_seconds_sum{scrape_job="random"} 0.20406449899999993 +prometheus_target_sync_length_seconds_count{scrape_job="random"} 2953 +# HELP prometheus_template_text_expansion_failures_total The total number of template text expansion failures. +# TYPE prometheus_template_text_expansion_failures_total counter +prometheus_template_text_expansion_failures_total 0 +# HELP prometheus_template_text_expansions_total The total number of template text expansions. +# TYPE prometheus_template_text_expansions_total counter +prometheus_template_text_expansions_total 2.126277e+06 +# HELP prometheus_treecache_watcher_goroutines The current number of watcher goroutines. +# TYPE prometheus_treecache_watcher_goroutines gauge +prometheus_treecache_watcher_goroutines 0 +# HELP prometheus_treecache_zookeeper_failures_total The total number of ZooKeeper failures. +# TYPE prometheus_treecache_zookeeper_failures_total counter +prometheus_treecache_zookeeper_failures_total 0 +# HELP prometheus_tsdb_blocks_loaded Number of currently loaded data blocks +# TYPE prometheus_tsdb_blocks_loaded gauge +prometheus_tsdb_blocks_loaded 17 +# HELP prometheus_tsdb_checkpoint_creations_failed_total Total number of checkpoint creations that failed. +# TYPE prometheus_tsdb_checkpoint_creations_failed_total counter +prometheus_tsdb_checkpoint_creations_failed_total 0 +# HELP prometheus_tsdb_checkpoint_creations_total Total number of checkpoint creations attempted. +# TYPE prometheus_tsdb_checkpoint_creations_total counter +prometheus_tsdb_checkpoint_creations_total 62 +# HELP prometheus_tsdb_checkpoint_deletions_failed_total Total number of checkpoint deletions that failed. +# TYPE prometheus_tsdb_checkpoint_deletions_failed_total counter +prometheus_tsdb_checkpoint_deletions_failed_total 0 +# HELP prometheus_tsdb_checkpoint_deletions_total Total number of checkpoint deletions attempted. +# TYPE prometheus_tsdb_checkpoint_deletions_total counter +prometheus_tsdb_checkpoint_deletions_total 62 +# HELP prometheus_tsdb_clean_start -1: lockfile is disabled. 0: a lockfile from a previous execution was replaced. 1: lockfile creation was clean +# TYPE prometheus_tsdb_clean_start gauge +prometheus_tsdb_clean_start 1 +# HELP prometheus_tsdb_compaction_chunk_range_seconds Final time range of chunks on their first compaction +# TYPE prometheus_tsdb_compaction_chunk_range_seconds histogram +prometheus_tsdb_compaction_chunk_range_seconds_bucket{le="100"} 86 +prometheus_tsdb_compaction_chunk_range_seconds_bucket{le="400"} 86 +prometheus_tsdb_compaction_chunk_range_seconds_bucket{le="1600"} 86 +prometheus_tsdb_compaction_chunk_range_seconds_bucket{le="6400"} 86 +prometheus_tsdb_compaction_chunk_range_seconds_bucket{le="25600"} 676 +prometheus_tsdb_compaction_chunk_range_seconds_bucket{le="102400"} 1555 +prometheus_tsdb_compaction_chunk_range_seconds_bucket{le="409600"} 2379 +prometheus_tsdb_compaction_chunk_range_seconds_bucket{le="1.6384e+06"} 34068 +prometheus_tsdb_compaction_chunk_range_seconds_bucket{le="6.5536e+06"} 4.819758e+06 +prometheus_tsdb_compaction_chunk_range_seconds_bucket{le="2.62144e+07"} 4.861956e+06 +prometheus_tsdb_compaction_chunk_range_seconds_bucket{le="+Inf"} 4.861956e+06 +prometheus_tsdb_compaction_chunk_range_seconds_sum 8.917524773366e+12 +prometheus_tsdb_compaction_chunk_range_seconds_count 4.861956e+06 +# HELP prometheus_tsdb_compaction_chunk_samples Final number of samples on their first compaction +# TYPE prometheus_tsdb_compaction_chunk_samples histogram +prometheus_tsdb_compaction_chunk_samples_bucket{le="4"} 1407 +prometheus_tsdb_compaction_chunk_samples_bucket{le="6"} 1544 +prometheus_tsdb_compaction_chunk_samples_bucket{le="9"} 1881 +prometheus_tsdb_compaction_chunk_samples_bucket{le="13.5"} 2065 +prometheus_tsdb_compaction_chunk_samples_bucket{le="20.25"} 2782 +prometheus_tsdb_compaction_chunk_samples_bucket{le="30.375"} 4342 +prometheus_tsdb_compaction_chunk_samples_bucket{le="45.5625"} 6180 +prometheus_tsdb_compaction_chunk_samples_bucket{le="68.34375"} 11518 +prometheus_tsdb_compaction_chunk_samples_bucket{le="102.515625"} 13890 +prometheus_tsdb_compaction_chunk_samples_bucket{le="153.7734375"} 4.810155e+06 +prometheus_tsdb_compaction_chunk_samples_bucket{le="230.66015625"} 4.86058e+06 +prometheus_tsdb_compaction_chunk_samples_bucket{le="345.990234375"} 4.861956e+06 +prometheus_tsdb_compaction_chunk_samples_bucket{le="+Inf"} 4.861956e+06 +prometheus_tsdb_compaction_chunk_samples_sum 5.86000708e+08 +prometheus_tsdb_compaction_chunk_samples_count 4.861956e+06 +# HELP prometheus_tsdb_compaction_chunk_size_bytes Final size of chunks on their first compaction +# TYPE prometheus_tsdb_compaction_chunk_size_bytes histogram +prometheus_tsdb_compaction_chunk_size_bytes_bucket{le="32"} 1233 +prometheus_tsdb_compaction_chunk_size_bytes_bucket{le="48"} 156238 +prometheus_tsdb_compaction_chunk_size_bytes_bucket{le="72"} 2.006456e+06 +prometheus_tsdb_compaction_chunk_size_bytes_bucket{le="108"} 3.568405e+06 +prometheus_tsdb_compaction_chunk_size_bytes_bucket{le="162"} 3.835144e+06 +prometheus_tsdb_compaction_chunk_size_bytes_bucket{le="243"} 4.034591e+06 +prometheus_tsdb_compaction_chunk_size_bytes_bucket{le="364.5"} 4.505646e+06 +prometheus_tsdb_compaction_chunk_size_bytes_bucket{le="546.75"} 4.69694e+06 +prometheus_tsdb_compaction_chunk_size_bytes_bucket{le="820.125"} 4.78551e+06 +prometheus_tsdb_compaction_chunk_size_bytes_bucket{le="1230.1875"} 4.861956e+06 +prometheus_tsdb_compaction_chunk_size_bytes_bucket{le="1845.28125"} 4.861956e+06 +prometheus_tsdb_compaction_chunk_size_bytes_bucket{le="2767.921875"} 4.861956e+06 +prometheus_tsdb_compaction_chunk_size_bytes_bucket{le="+Inf"} 4.861956e+06 +prometheus_tsdb_compaction_chunk_size_bytes_sum 6.81852972e+08 +prometheus_tsdb_compaction_chunk_size_bytes_count 4.861956e+06 +# HELP prometheus_tsdb_compaction_duration_seconds Duration of compaction runs +# TYPE prometheus_tsdb_compaction_duration_seconds histogram +prometheus_tsdb_compaction_duration_seconds_bucket{le="1"} 61 +prometheus_tsdb_compaction_duration_seconds_bucket{le="2"} 155 +prometheus_tsdb_compaction_duration_seconds_bucket{le="4"} 180 +prometheus_tsdb_compaction_duration_seconds_bucket{le="8"} 183 +prometheus_tsdb_compaction_duration_seconds_bucket{le="16"} 183 +prometheus_tsdb_compaction_duration_seconds_bucket{le="32"} 183 +prometheus_tsdb_compaction_duration_seconds_bucket{le="64"} 183 +prometheus_tsdb_compaction_duration_seconds_bucket{le="128"} 183 +prometheus_tsdb_compaction_duration_seconds_bucket{le="256"} 183 +prometheus_tsdb_compaction_duration_seconds_bucket{le="512"} 183 +prometheus_tsdb_compaction_duration_seconds_bucket{le="1024"} 183 +prometheus_tsdb_compaction_duration_seconds_bucket{le="2048"} 183 +prometheus_tsdb_compaction_duration_seconds_bucket{le="4096"} 183 +prometheus_tsdb_compaction_duration_seconds_bucket{le="8192"} 183 +prometheus_tsdb_compaction_duration_seconds_bucket{le="+Inf"} 183 +prometheus_tsdb_compaction_duration_seconds_sum 254.9095696899999 +prometheus_tsdb_compaction_duration_seconds_count 183 +# HELP prometheus_tsdb_compaction_populating_block Set to 1 when a block is currently being written to the disk. +# TYPE prometheus_tsdb_compaction_populating_block gauge +prometheus_tsdb_compaction_populating_block 0 +# HELP prometheus_tsdb_compactions_failed_total Total number of compactions that failed for the partition. +# TYPE prometheus_tsdb_compactions_failed_total counter +prometheus_tsdb_compactions_failed_total 0 +# HELP prometheus_tsdb_compactions_skipped_total Total number of skipped compactions due to disabled auto compaction. +# TYPE prometheus_tsdb_compactions_skipped_total counter +prometheus_tsdb_compactions_skipped_total 0 +# HELP prometheus_tsdb_compactions_total Total number of compactions that were executed for the partition. +# TYPE prometheus_tsdb_compactions_total counter +prometheus_tsdb_compactions_total 183 +# HELP prometheus_tsdb_compactions_triggered_total Total number of triggered compactions for the partition. +# TYPE prometheus_tsdb_compactions_triggered_total counter +prometheus_tsdb_compactions_triggered_total 14855 +# HELP prometheus_tsdb_data_replay_duration_seconds Time taken to replay the data on disk. +# TYPE prometheus_tsdb_data_replay_duration_seconds gauge +prometheus_tsdb_data_replay_duration_seconds 5.550163617 +# HELP prometheus_tsdb_exemplar_exemplars_appended_total Total number of appended exemplars. +# TYPE prometheus_tsdb_exemplar_exemplars_appended_total counter +prometheus_tsdb_exemplar_exemplars_appended_total 0 +# HELP prometheus_tsdb_exemplar_exemplars_in_storage Number of exemplars currently in circular storage. +# TYPE prometheus_tsdb_exemplar_exemplars_in_storage gauge +prometheus_tsdb_exemplar_exemplars_in_storage 0 +# HELP prometheus_tsdb_exemplar_last_exemplars_timestamp_seconds The timestamp of the oldest exemplar stored in circular storage. Useful to check for what timerange the current exemplar buffer limit allows. This usually means the last timestampfor all exemplars for a typical setup. This is not true though if one of the series timestamp is in future compared to rest series. +# TYPE prometheus_tsdb_exemplar_last_exemplars_timestamp_seconds gauge +prometheus_tsdb_exemplar_last_exemplars_timestamp_seconds 0 +# HELP prometheus_tsdb_exemplar_max_exemplars Total number of exemplars the exemplar storage can store, resizeable. +# TYPE prometheus_tsdb_exemplar_max_exemplars gauge +prometheus_tsdb_exemplar_max_exemplars 0 +# HELP prometheus_tsdb_exemplar_out_of_order_exemplars_total Total number of out of order exemplar ingestion failed attempts. +# TYPE prometheus_tsdb_exemplar_out_of_order_exemplars_total counter +prometheus_tsdb_exemplar_out_of_order_exemplars_total 0 +# HELP prometheus_tsdb_exemplar_series_with_exemplars_in_storage Number of series with exemplars currently in circular storage. +# TYPE prometheus_tsdb_exemplar_series_with_exemplars_in_storage gauge +prometheus_tsdb_exemplar_series_with_exemplars_in_storage 0 +# HELP prometheus_tsdb_head_active_appenders Number of currently active appender transactions +# TYPE prometheus_tsdb_head_active_appenders gauge +prometheus_tsdb_head_active_appenders 0 +# HELP prometheus_tsdb_head_chunks Total number of chunks in the head block. +# TYPE prometheus_tsdb_head_chunks gauge +prometheus_tsdb_head_chunks 31476 +# HELP prometheus_tsdb_head_chunks_created_total Total number of chunks created in the head +# TYPE prometheus_tsdb_head_chunks_created_total counter +prometheus_tsdb_head_chunks_created_total 4.893432e+06 +# HELP prometheus_tsdb_head_chunks_removed_total Total number of chunks removed in the head +# TYPE prometheus_tsdb_head_chunks_removed_total counter +prometheus_tsdb_head_chunks_removed_total 4.861956e+06 +# HELP prometheus_tsdb_head_chunks_storage_size_bytes Size of the chunks_head directory. +# TYPE prometheus_tsdb_head_chunks_storage_size_bytes gauge +prometheus_tsdb_head_chunks_storage_size_bytes 7.237299e+06 +# HELP prometheus_tsdb_head_gc_duration_seconds Runtime of garbage collection in the head block. +# TYPE prometheus_tsdb_head_gc_duration_seconds summary +prometheus_tsdb_head_gc_duration_seconds_sum 4.773801686000001 +prometheus_tsdb_head_gc_duration_seconds_count 123 +# HELP prometheus_tsdb_head_max_time Maximum timestamp of the head block. The unit is decided by the library consumer. +# TYPE prometheus_tsdb_head_max_time gauge +prometheus_tsdb_head_max_time 1.738949375191e+12 +# HELP prometheus_tsdb_head_max_time_seconds Maximum timestamp of the head block. +# TYPE prometheus_tsdb_head_max_time_seconds gauge +prometheus_tsdb_head_max_time_seconds 1.738949375e+09 +# HELP prometheus_tsdb_head_min_time Minimum time bound of the head block. The unit is decided by the library consumer. +# TYPE prometheus_tsdb_head_min_time gauge +prometheus_tsdb_head_min_time 1.738944000171e+12 +# HELP prometheus_tsdb_head_min_time_seconds Minimum time bound of the head block. +# TYPE prometheus_tsdb_head_min_time_seconds gauge +prometheus_tsdb_head_min_time_seconds 1.738944e+09 +# HELP prometheus_tsdb_head_out_of_order_samples_appended_total Total number of appended out of order samples. +# TYPE prometheus_tsdb_head_out_of_order_samples_appended_total counter +prometheus_tsdb_head_out_of_order_samples_appended_total{type="float"} 0 +prometheus_tsdb_head_out_of_order_samples_appended_total{type="histogram"} 0 +# HELP prometheus_tsdb_head_samples_appended_total Total number of appended samples. +# TYPE prometheus_tsdb_head_samples_appended_total counter +prometheus_tsdb_head_samples_appended_total{type="float"} 5.85543187e+08 +prometheus_tsdb_head_samples_appended_total{type="histogram"} 0 +# HELP prometheus_tsdb_head_series Total number of series in the head block. +# TYPE prometheus_tsdb_head_series gauge +prometheus_tsdb_head_series 10720 +# HELP prometheus_tsdb_head_series_created_total Total number of series created in the head +# TYPE prometheus_tsdb_head_series_created_total counter +prometheus_tsdb_head_series_created_total 18541 +# HELP prometheus_tsdb_head_series_not_found_total Total number of requests for series that were not found. +# TYPE prometheus_tsdb_head_series_not_found_total counter +prometheus_tsdb_head_series_not_found_total 0 +# HELP prometheus_tsdb_head_series_removed_total Total number of series removed in the head +# TYPE prometheus_tsdb_head_series_removed_total counter +prometheus_tsdb_head_series_removed_total 7821 +# HELP prometheus_tsdb_head_truncations_failed_total Total number of head truncations that failed. +# TYPE prometheus_tsdb_head_truncations_failed_total counter +prometheus_tsdb_head_truncations_failed_total 0 +# HELP prometheus_tsdb_head_truncations_total Total number of head truncations attempted. +# TYPE prometheus_tsdb_head_truncations_total counter +prometheus_tsdb_head_truncations_total 123 +# HELP prometheus_tsdb_isolation_high_watermark The highest TSDB append ID that has been given out. +# TYPE prometheus_tsdb_isolation_high_watermark gauge +prometheus_tsdb_isolation_high_watermark 7.852949e+06 +# HELP prometheus_tsdb_isolation_low_watermark The lowest TSDB append ID that is still referenced. +# TYPE prometheus_tsdb_isolation_low_watermark gauge +prometheus_tsdb_isolation_low_watermark 7.852949e+06 +# HELP prometheus_tsdb_lowest_timestamp Lowest timestamp value stored in the database. The unit is decided by the library consumer. +# TYPE prometheus_tsdb_lowest_timestamp gauge +prometheus_tsdb_lowest_timestamp 1.73618640004e+12 +# HELP prometheus_tsdb_lowest_timestamp_seconds Lowest timestamp value stored in the database. +# TYPE prometheus_tsdb_lowest_timestamp_seconds gauge +prometheus_tsdb_lowest_timestamp_seconds 1.7361864e+09 +# HELP prometheus_tsdb_mmap_chunk_corruptions_total Total number of memory-mapped chunk corruptions. +# TYPE prometheus_tsdb_mmap_chunk_corruptions_total counter +prometheus_tsdb_mmap_chunk_corruptions_total 0 +# HELP prometheus_tsdb_mmap_chunks_total Total number of chunks that were memory-mapped. +# TYPE prometheus_tsdb_mmap_chunks_total counter +prometheus_tsdb_mmap_chunks_total 4.851264e+06 +# HELP prometheus_tsdb_out_of_bound_samples_total Total number of out of bound samples ingestion failed attempts with out of order support disabled. +# TYPE prometheus_tsdb_out_of_bound_samples_total counter +prometheus_tsdb_out_of_bound_samples_total{type="float"} 0 +# HELP prometheus_tsdb_out_of_order_samples_total Total number of out of order samples ingestion failed attempts due to out of order being disabled. +# TYPE prometheus_tsdb_out_of_order_samples_total counter +prometheus_tsdb_out_of_order_samples_total{type="float"} 517 +prometheus_tsdb_out_of_order_samples_total{type="histogram"} 0 +# HELP prometheus_tsdb_reloads_failures_total Number of times the database failed to reloadBlocks block data from disk. +# TYPE prometheus_tsdb_reloads_failures_total counter +prometheus_tsdb_reloads_failures_total 0 +# HELP prometheus_tsdb_reloads_total Number of times the database reloaded block data from disk. +# TYPE prometheus_tsdb_reloads_total counter +prometheus_tsdb_reloads_total 14822 +# HELP prometheus_tsdb_retention_limit_bytes Max number of bytes to be retained in the tsdb blocks, configured 0 means disabled +# TYPE prometheus_tsdb_retention_limit_bytes gauge +prometheus_tsdb_retention_limit_bytes 0 +# HELP prometheus_tsdb_retention_limit_seconds How long to retain samples in storage. +# TYPE prometheus_tsdb_retention_limit_seconds gauge +prometheus_tsdb_retention_limit_seconds 2.6784e+06 +# HELP prometheus_tsdb_size_retentions_total The number of times that blocks were deleted because the maximum number of bytes was exceeded. +# TYPE prometheus_tsdb_size_retentions_total counter +prometheus_tsdb_size_retentions_total 0 +# HELP prometheus_tsdb_snapshot_replay_error_total Total number snapshot replays that failed. +# TYPE prometheus_tsdb_snapshot_replay_error_total counter +prometheus_tsdb_snapshot_replay_error_total 0 +# HELP prometheus_tsdb_storage_blocks_bytes The number of bytes that are currently used for local storage by all blocks. +# TYPE prometheus_tsdb_storage_blocks_bytes gauge +prometheus_tsdb_storage_blocks_bytes 2.762863592e+09 +# HELP prometheus_tsdb_symbol_table_size_bytes Size of symbol table in memory for loaded blocks +# TYPE prometheus_tsdb_symbol_table_size_bytes gauge +prometheus_tsdb_symbol_table_size_bytes 10616 +# HELP prometheus_tsdb_time_retentions_total The number of times that blocks were deleted because the maximum time limit was exceeded. +# TYPE prometheus_tsdb_time_retentions_total counter +prometheus_tsdb_time_retentions_total 5 +# HELP prometheus_tsdb_tombstone_cleanup_seconds The time taken to recompact blocks to remove tombstones. +# TYPE prometheus_tsdb_tombstone_cleanup_seconds histogram +prometheus_tsdb_tombstone_cleanup_seconds_bucket{le="+Inf"} 0 +prometheus_tsdb_tombstone_cleanup_seconds_sum 0 +prometheus_tsdb_tombstone_cleanup_seconds_count 0 +# HELP prometheus_tsdb_too_old_samples_total Total number of out of order samples ingestion failed attempts with out of support enabled, but sample outside of time window. +# TYPE prometheus_tsdb_too_old_samples_total counter +prometheus_tsdb_too_old_samples_total{type="float"} 0 +# HELP prometheus_tsdb_vertical_compactions_total Total number of compactions done on overlapping blocks. +# TYPE prometheus_tsdb_vertical_compactions_total counter +prometheus_tsdb_vertical_compactions_total 0 +# HELP prometheus_tsdb_wal_completed_pages_total Total number of completed pages. +# TYPE prometheus_tsdb_wal_completed_pages_total counter +prometheus_tsdb_wal_completed_pages_total 109271 +# HELP prometheus_tsdb_wal_corruptions_total Total number of WAL corruptions. +# TYPE prometheus_tsdb_wal_corruptions_total counter +prometheus_tsdb_wal_corruptions_total 0 +# HELP prometheus_tsdb_wal_fsync_duration_seconds Duration of write log fsync. +# TYPE prometheus_tsdb_wal_fsync_duration_seconds summary +prometheus_tsdb_wal_fsync_duration_seconds{quantile="0.5"} NaN +prometheus_tsdb_wal_fsync_duration_seconds{quantile="0.9"} NaN +prometheus_tsdb_wal_fsync_duration_seconds{quantile="0.99"} NaN +prometheus_tsdb_wal_fsync_duration_seconds_sum 4.842524568000002 +prometheus_tsdb_wal_fsync_duration_seconds_count 123 +# HELP prometheus_tsdb_wal_page_flushes_total Total number of page flushes. +# TYPE prometheus_tsdb_wal_page_flushes_total counter +prometheus_tsdb_wal_page_flushes_total 2.293951e+06 +# HELP prometheus_tsdb_wal_segment_current Write log segment index that TSDB is currently writing to. +# TYPE prometheus_tsdb_wal_segment_current gauge +prometheus_tsdb_wal_segment_current 24726 +# HELP prometheus_tsdb_wal_storage_size_bytes Size of the write log directory. +# TYPE prometheus_tsdb_wal_storage_size_bytes gauge +prometheus_tsdb_wal_storage_size_bytes 6.9168385e+07 +# HELP prometheus_tsdb_wal_truncate_duration_seconds Duration of WAL truncation. +# TYPE prometheus_tsdb_wal_truncate_duration_seconds summary +prometheus_tsdb_wal_truncate_duration_seconds_sum 121.61954577099996 +prometheus_tsdb_wal_truncate_duration_seconds_count 62 +# HELP prometheus_tsdb_wal_truncations_failed_total Total number of write log truncations that failed. +# TYPE prometheus_tsdb_wal_truncations_failed_total counter +prometheus_tsdb_wal_truncations_failed_total 0 +# HELP prometheus_tsdb_wal_truncations_total Total number of write log truncations attempted. +# TYPE prometheus_tsdb_wal_truncations_total counter +prometheus_tsdb_wal_truncations_total 62 +# HELP prometheus_tsdb_wal_writes_failed_total Total number of write log writes that failed. +# TYPE prometheus_tsdb_wal_writes_failed_total counter +prometheus_tsdb_wal_writes_failed_total 0 +# HELP prometheus_web_federation_errors_total Total number of errors that occurred while sending federation responses. +# TYPE prometheus_web_federation_errors_total counter +prometheus_web_federation_errors_total 0 +# HELP prometheus_web_federation_warnings_total Total number of warnings that occurred while sending federation responses. +# TYPE prometheus_web_federation_warnings_total counter +prometheus_web_federation_warnings_total 0 +# HELP promhttp_metric_handler_requests_in_flight Current number of scrapes being served. +# TYPE promhttp_metric_handler_requests_in_flight gauge +promhttp_metric_handler_requests_in_flight 1 +# HELP promhttp_metric_handler_requests_total Total number of scrapes by HTTP status code. +# TYPE promhttp_metric_handler_requests_total counter +promhttp_metric_handler_requests_total{code="200"} 4.059092e+06 +promhttp_metric_handler_requests_total{code="500"} 0 +promhttp_metric_handler_requests_total{code="503"} 0 +# EOF diff --git a/model/textparse/testdata/omtestdata.txt b/model/textparse/testdata/alltypes.5mfs.om.txt similarity index 100% rename from model/textparse/testdata/omtestdata.txt rename to model/textparse/testdata/alltypes.5mfs.om.txt diff --git a/model/textparse/testdata/promtestdata.nometa.txt b/model/textparse/testdata/promtestdata.nometa.txt deleted file mode 100644 index 235f0aa464..0000000000 --- a/model/textparse/testdata/promtestdata.nometa.txt +++ /dev/null @@ -1,411 +0,0 @@ -go_gc_duration_seconds{quantile="0"} 4.9351e-05 -go_gc_duration_seconds{quantile="0.25"} 7.424100000000001e-05 -go_gc_duration_seconds{quantile="0.5"} 8.3835e-05 -go_gc_duration_seconds{quantile="0.75"} 0.000106744 -go_gc_duration_seconds{quantile="1"} 0.002072195 -go_gc_duration_seconds_sum 0.012139815 -go_gc_duration_seconds_count 99 -go_goroutines 33 -go_memstats_alloc_bytes 1.7518624e+07 -go_memstats_alloc_bytes_total 8.3062296e+08 -go_memstats_buck_hash_sys_bytes 1.494637e+06 -go_memstats_frees_total 4.65658e+06 -go_memstats_gc_sys_bytes 1.107968e+06 -go_memstats_heap_alloc_bytes 1.7518624e+07 -go_memstats_heap_idle_bytes 6.668288e+06 -go_memstats_heap_inuse_bytes 1.8956288e+07 -go_memstats_heap_objects 72755 -go_memstats_heap_released_bytes_total 0 -go_memstats_heap_sys_bytes 2.5624576e+07 -go_memstats_last_gc_time_seconds 1.4843955586166437e+09 -go_memstats_lookups_total 2089 -go_memstats_mallocs_total 4.729335e+06 -go_memstats_mcache_inuse_bytes 9600 -go_memstats_mcache_sys_bytes 16384 -go_memstats_mspan_inuse_bytes 211520 -go_memstats_mspan_sys_bytes 245760 -go_memstats_next_gc_bytes 2.033527e+07 -go_memstats_other_sys_bytes 2.077323e+06 -go_memstats_stack_inuse_bytes 1.6384e+06 -go_memstats_stack_sys_bytes 1.6384e+06 -go_memstats_sys_bytes 3.2205048e+07 -http_request_duration_microseconds{handler="alerts",quantile="0.5"} NaN -http_request_duration_microseconds{handler="alerts",quantile="0.9"} NaN -http_request_duration_microseconds{handler="alerts",quantile="0.99"} NaN -http_request_duration_microseconds_sum{handler="alerts"} 0 -http_request_duration_microseconds_count{handler="alerts"} 0 -http_request_duration_microseconds{handler="config",quantile="0.5"} NaN -http_request_duration_microseconds{handler="config",quantile="0.9"} NaN -http_request_duration_microseconds{handler="config",quantile="0.99"} NaN -http_request_duration_microseconds_sum{handler="config"} 0 -http_request_duration_microseconds_count{handler="config"} 0 -http_request_duration_microseconds{handler="consoles",quantile="0.5"} NaN -http_request_duration_microseconds{handler="consoles",quantile="0.9"} NaN -http_request_duration_microseconds{handler="consoles",quantile="0.99"} NaN -http_request_duration_microseconds_sum{handler="consoles"} 0 -http_request_duration_microseconds_count{handler="consoles"} 0 -http_request_duration_microseconds{handler="drop_series",quantile="0.5"} NaN -http_request_duration_microseconds{handler="drop_series",quantile="0.9"} NaN -http_request_duration_microseconds{handler="drop_series",quantile="0.99"} NaN -http_request_duration_microseconds_sum{handler="drop_series"} 0 -http_request_duration_microseconds_count{handler="drop_series"} 0 -http_request_duration_microseconds{handler="federate",quantile="0.5"} NaN -http_request_duration_microseconds{handler="federate",quantile="0.9"} NaN -http_request_duration_microseconds{handler="federate",quantile="0.99"} NaN -http_request_duration_microseconds_sum{handler="federate"} 0 -http_request_duration_microseconds_count{handler="federate"} 0 -http_request_duration_microseconds{handler="flags",quantile="0.5"} NaN -http_request_duration_microseconds{handler="flags",quantile="0.9"} NaN -http_request_duration_microseconds{handler="flags",quantile="0.99"} NaN -http_request_duration_microseconds_sum{handler="flags"} 0 -http_request_duration_microseconds_count{handler="flags"} 0 -http_request_duration_microseconds{handler="graph",quantile="0.5"} 771.655 -http_request_duration_microseconds{handler="graph",quantile="0.9"} 1761.823 -http_request_duration_microseconds{handler="graph",quantile="0.99"} 1761.823 -http_request_duration_microseconds_sum{handler="graph"} 5803.93 -http_request_duration_microseconds_count{handler="graph"} 3 -http_request_duration_microseconds{handler="heap",quantile="0.5"} NaN -http_request_duration_microseconds{handler="heap",quantile="0.9"} NaN -http_request_duration_microseconds{handler="heap",quantile="0.99"} NaN -http_request_duration_microseconds_sum{handler="heap"} 0 -http_request_duration_microseconds_count{handler="heap"} 0 -http_request_duration_microseconds{handler="label_values",quantile="0.5"} 325.401 -http_request_duration_microseconds{handler="label_values",quantile="0.9"} 414.708 -http_request_duration_microseconds{handler="label_values",quantile="0.99"} 414.708 -http_request_duration_microseconds_sum{handler="label_values"} 3995.574 -http_request_duration_microseconds_count{handler="label_values"} 3 -http_request_duration_microseconds{handler="options",quantile="0.5"} NaN -http_request_duration_microseconds{handler="options",quantile="0.9"} NaN -http_request_duration_microseconds{handler="options",quantile="0.99"} NaN -http_request_duration_microseconds_sum{handler="options"} 0 -http_request_duration_microseconds_count{handler="options"} 0 -http_request_duration_microseconds{handler="prometheus",quantile="0.5"} 1351.859 -http_request_duration_microseconds{handler="prometheus",quantile="0.9"} 1714.035 -http_request_duration_microseconds{handler="prometheus",quantile="0.99"} 2833.523 -http_request_duration_microseconds_sum{handler="prometheus"} 661851.54 -http_request_duration_microseconds_count{handler="prometheus"} 462 -http_request_duration_microseconds{handler="query",quantile="0.5"} 3885.448 -http_request_duration_microseconds{handler="query",quantile="0.9"} 4390.558 -http_request_duration_microseconds{handler="query",quantile="0.99"} 4390.558 -http_request_duration_microseconds_sum{handler="query"} 26074.11 -http_request_duration_microseconds_count{handler="query"} 6 -http_request_duration_microseconds{handler="query_range",quantile="0.5"} NaN -http_request_duration_microseconds{handler="query_range",quantile="0.9"} NaN -http_request_duration_microseconds{handler="query_range",quantile="0.99"} NaN -http_request_duration_microseconds_sum{handler="query_range"} 0 -http_request_duration_microseconds_count{handler="query_range"} 0 -http_request_duration_microseconds{handler="rules",quantile="0.5"} NaN -http_request_duration_microseconds{handler="rules",quantile="0.9"} NaN -http_request_duration_microseconds{handler="rules",quantile="0.99"} NaN -http_request_duration_microseconds_sum{handler="rules"} 0 -http_request_duration_microseconds_count{handler="rules"} 0 -http_request_duration_microseconds{handler="series",quantile="0.5"} NaN -http_request_duration_microseconds{handler="series",quantile="0.9"} NaN -http_request_duration_microseconds{handler="series",quantile="0.99"} NaN -http_request_duration_microseconds_sum{handler="series"} 0 -http_request_duration_microseconds_count{handler="series"} 0 -http_request_duration_microseconds{handler="static",quantile="0.5"} 212.311 -http_request_duration_microseconds{handler="static",quantile="0.9"} 265.174 -http_request_duration_microseconds{handler="static",quantile="0.99"} 265.174 -http_request_duration_microseconds_sum{handler="static"} 6458.621 -http_request_duration_microseconds_count{handler="static"} 3 -http_request_duration_microseconds{handler="status",quantile="0.5"} NaN -http_request_duration_microseconds{handler="status",quantile="0.9"} NaN -http_request_duration_microseconds{handler="status",quantile="0.99"} NaN -http_request_duration_microseconds_sum{handler="status"} 0 -http_request_duration_microseconds_count{handler="status"} 0 -http_request_duration_microseconds{handler="targets",quantile="0.5"} NaN -http_request_duration_microseconds{handler="targets",quantile="0.9"} NaN -http_request_duration_microseconds{handler="targets",quantile="0.99"} NaN -http_request_duration_microseconds_sum{handler="targets"} 0 -http_request_duration_microseconds_count{handler="targets"} 0 -http_request_duration_microseconds{handler="version",quantile="0.5"} NaN -http_request_duration_microseconds{handler="version",quantile="0.9"} NaN -http_request_duration_microseconds{handler="version",quantile="0.99"} NaN -http_request_duration_microseconds_sum{handler="version"} 0 -http_request_duration_microseconds_count{handler="version"} 0 -http_request_size_bytes{handler="alerts",quantile="0.5"} NaN -http_request_size_bytes{handler="alerts",quantile="0.9"} NaN -http_request_size_bytes{handler="alerts",quantile="0.99"} NaN -http_request_size_bytes_sum{handler="alerts"} 0 -http_request_size_bytes_count{handler="alerts"} 0 -http_request_size_bytes{handler="config",quantile="0.5"} NaN -http_request_size_bytes{handler="config",quantile="0.9"} NaN -http_request_size_bytes{handler="config",quantile="0.99"} NaN -http_request_size_bytes_sum{handler="config"} 0 -http_request_size_bytes_count{handler="config"} 0 -http_request_size_bytes{handler="consoles",quantile="0.5"} NaN -http_request_size_bytes{handler="consoles",quantile="0.9"} NaN -http_request_size_bytes{handler="consoles",quantile="0.99"} NaN -http_request_size_bytes_sum{handler="consoles"} 0 -http_request_size_bytes_count{handler="consoles"} 0 -http_request_size_bytes{handler="drop_series",quantile="0.5"} NaN -http_request_size_bytes{handler="drop_series",quantile="0.9"} NaN -http_request_size_bytes{handler="drop_series",quantile="0.99"} NaN -http_request_size_bytes_sum{handler="drop_series"} 0 -http_request_size_bytes_count{handler="drop_series"} 0 -http_request_size_bytes{handler="federate",quantile="0.5"} NaN -http_request_size_bytes{handler="federate",quantile="0.9"} NaN -http_request_size_bytes{handler="federate",quantile="0.99"} NaN -http_request_size_bytes_sum{handler="federate"} 0 -http_request_size_bytes_count{handler="federate"} 0 -http_request_size_bytes{handler="flags",quantile="0.5"} NaN -http_request_size_bytes{handler="flags",quantile="0.9"} NaN -http_request_size_bytes{handler="flags",quantile="0.99"} NaN -http_request_size_bytes_sum{handler="flags"} 0 -http_request_size_bytes_count{handler="flags"} 0 -http_request_size_bytes{handler="graph",quantile="0.5"} 367 -http_request_size_bytes{handler="graph",quantile="0.9"} 389 -http_request_size_bytes{handler="graph",quantile="0.99"} 389 -http_request_size_bytes_sum{handler="graph"} 1145 -http_request_size_bytes_count{handler="graph"} 3 -http_request_size_bytes{handler="heap",quantile="0.5"} NaN -http_request_size_bytes{handler="heap",quantile="0.9"} NaN -http_request_size_bytes{handler="heap",quantile="0.99"} NaN -http_request_size_bytes_sum{handler="heap"} 0 -http_request_size_bytes_count{handler="heap"} 0 -http_request_size_bytes{handler="label_values",quantile="0.5"} 416 -http_request_size_bytes{handler="label_values",quantile="0.9"} 416 -http_request_size_bytes{handler="label_values",quantile="0.99"} 416 -http_request_size_bytes_sum{handler="label_values"} 1248 -http_request_size_bytes_count{handler="label_values"} 3 -http_request_size_bytes{handler="options",quantile="0.5"} NaN -http_request_size_bytes{handler="options",quantile="0.9"} NaN -http_request_size_bytes{handler="options",quantile="0.99"} NaN -http_request_size_bytes_sum{handler="options"} 0 -http_request_size_bytes_count{handler="options"} 0 -http_request_size_bytes{handler="prometheus",quantile="0.5"} 238 -http_request_size_bytes{handler="prometheus",quantile="0.9"} 238 -http_request_size_bytes{handler="prometheus",quantile="0.99"} 238 -http_request_size_bytes_sum{handler="prometheus"} 109956 -http_request_size_bytes_count{handler="prometheus"} 462 -http_request_size_bytes{handler="query",quantile="0.5"} 531 -http_request_size_bytes{handler="query",quantile="0.9"} 531 -http_request_size_bytes{handler="query",quantile="0.99"} 531 -http_request_size_bytes_sum{handler="query"} 3186 -http_request_size_bytes_count{handler="query"} 6 -http_request_size_bytes{handler="query_range",quantile="0.5"} NaN -http_request_size_bytes{handler="query_range",quantile="0.9"} NaN -http_request_size_bytes{handler="query_range",quantile="0.99"} NaN -http_request_size_bytes_sum{handler="query_range"} 0 -http_request_size_bytes_count{handler="query_range"} 0 -http_request_size_bytes{handler="rules",quantile="0.5"} NaN -http_request_size_bytes{handler="rules",quantile="0.9"} NaN -http_request_size_bytes{handler="rules",quantile="0.99"} NaN -http_request_size_bytes_sum{handler="rules"} 0 -http_request_size_bytes_count{handler="rules"} 0 -http_request_size_bytes{handler="series",quantile="0.5"} NaN -http_request_size_bytes{handler="series",quantile="0.9"} NaN -http_request_size_bytes{handler="series",quantile="0.99"} NaN -http_request_size_bytes_sum{handler="series"} 0 -http_request_size_bytes_count{handler="series"} 0 -http_request_size_bytes{handler="static",quantile="0.5"} 379 -http_request_size_bytes{handler="static",quantile="0.9"} 379 -http_request_size_bytes{handler="static",quantile="0.99"} 379 -http_request_size_bytes_sum{handler="static"} 1137 -http_request_size_bytes_count{handler="static"} 3 -http_request_size_bytes{handler="status",quantile="0.5"} NaN -http_request_size_bytes{handler="status",quantile="0.9"} NaN -http_request_size_bytes{handler="status",quantile="0.99"} NaN -http_request_size_bytes_sum{handler="status"} 0 -http_request_size_bytes_count{handler="status"} 0 -http_request_size_bytes{handler="targets",quantile="0.5"} NaN -http_request_size_bytes{handler="targets",quantile="0.9"} NaN -http_request_size_bytes{handler="targets",quantile="0.99"} NaN -http_request_size_bytes_sum{handler="targets"} 0 -http_request_size_bytes_count{handler="targets"} 0 -http_request_size_bytes{handler="version",quantile="0.5"} NaN -http_request_size_bytes{handler="version",quantile="0.9"} NaN -http_request_size_bytes{handler="version",quantile="0.99"} NaN -http_request_size_bytes_sum{handler="version"} 0 -http_request_size_bytes_count{handler="version"} 0 -http_requests_total{code="200",handler="graph",method="get"} 3 -http_requests_total{code="200",handler="label_values",method="get"} 3 -http_requests_total{code="200",handler="prometheus",method="get"} 462 -http_requests_total{code="200",handler="query",method="get"} 6 -http_requests_total{code="200",handler="static",method="get"} 3 -http_response_size_bytes{handler="alerts",quantile="0.5"} NaN -http_response_size_bytes{handler="alerts",quantile="0.9"} NaN -http_response_size_bytes{handler="alerts",quantile="0.99"} NaN -http_response_size_bytes_sum{handler="alerts"} 0 -http_response_size_bytes_count{handler="alerts"} 0 -http_response_size_bytes{handler="config",quantile="0.5"} NaN -http_response_size_bytes{handler="config",quantile="0.9"} NaN -http_response_size_bytes{handler="config",quantile="0.99"} NaN -http_response_size_bytes_sum{handler="config"} 0 -http_response_size_bytes_count{handler="config"} 0 -http_response_size_bytes{handler="consoles",quantile="0.5"} NaN -http_response_size_bytes{handler="consoles",quantile="0.9"} NaN -http_response_size_bytes{handler="consoles",quantile="0.99"} NaN -http_response_size_bytes_sum{handler="consoles"} 0 -http_response_size_bytes_count{handler="consoles"} 0 -http_response_size_bytes{handler="drop_series",quantile="0.5"} NaN -http_response_size_bytes{handler="drop_series",quantile="0.9"} NaN -http_response_size_bytes{handler="drop_series",quantile="0.99"} NaN -http_response_size_bytes_sum{handler="drop_series"} 0 -http_response_size_bytes_count{handler="drop_series"} 0 -http_response_size_bytes{handler="federate",quantile="0.5"} NaN -http_response_size_bytes{handler="federate",quantile="0.9"} NaN -http_response_size_bytes{handler="federate",quantile="0.99"} NaN -http_response_size_bytes_sum{handler="federate"} 0 -http_response_size_bytes_count{handler="federate"} 0 -http_response_size_bytes{handler="flags",quantile="0.5"} NaN -http_response_size_bytes{handler="flags",quantile="0.9"} NaN -http_response_size_bytes{handler="flags",quantile="0.99"} NaN -http_response_size_bytes_sum{handler="flags"} 0 -http_response_size_bytes_count{handler="flags"} 0 -http_response_size_bytes{handler="graph",quantile="0.5"} 3619 -http_response_size_bytes{handler="graph",quantile="0.9"} 3619 -http_response_size_bytes{handler="graph",quantile="0.99"} 3619 -http_response_size_bytes_sum{handler="graph"} 10857 -http_response_size_bytes_count{handler="graph"} 3 -http_response_size_bytes{handler="heap",quantile="0.5"} NaN -http_response_size_bytes{handler="heap",quantile="0.9"} NaN -http_response_size_bytes{handler="heap",quantile="0.99"} NaN -http_response_size_bytes_sum{handler="heap"} 0 -http_response_size_bytes_count{handler="heap"} 0 -http_response_size_bytes{handler="label_values",quantile="0.5"} 642 -http_response_size_bytes{handler="label_values",quantile="0.9"} 642 -http_response_size_bytes{handler="label_values",quantile="0.99"} 642 -http_response_size_bytes_sum{handler="label_values"} 1926 -http_response_size_bytes_count{handler="label_values"} 3 -http_response_size_bytes{handler="options",quantile="0.5"} NaN -http_response_size_bytes{handler="options",quantile="0.9"} NaN -http_response_size_bytes{handler="options",quantile="0.99"} NaN -http_response_size_bytes_sum{handler="options"} 0 -http_response_size_bytes_count{handler="options"} 0 -http_response_size_bytes{handler="prometheus",quantile="0.5"} 3033 -http_response_size_bytes{handler="prometheus",quantile="0.9"} 3123 -http_response_size_bytes{handler="prometheus",quantile="0.99"} 3128 -http_response_size_bytes_sum{handler="prometheus"} 1.374097e+06 -http_response_size_bytes_count{handler="prometheus"} 462 -http_response_size_bytes{handler="query",quantile="0.5"} 776 -http_response_size_bytes{handler="query",quantile="0.9"} 781 -http_response_size_bytes{handler="query",quantile="0.99"} 781 -http_response_size_bytes_sum{handler="query"} 4656 -http_response_size_bytes_count{handler="query"} 6 -http_response_size_bytes{handler="query_range",quantile="0.5"} NaN -http_response_size_bytes{handler="query_range",quantile="0.9"} NaN -http_response_size_bytes{handler="query_range",quantile="0.99"} NaN -http_response_size_bytes_sum{handler="query_range"} 0 -http_response_size_bytes_count{handler="query_range"} 0 -http_response_size_bytes{handler="rules",quantile="0.5"} NaN -http_response_size_bytes{handler="rules",quantile="0.9"} NaN -http_response_size_bytes{handler="rules",quantile="0.99"} NaN -http_response_size_bytes_sum{handler="rules"} 0 -http_response_size_bytes_count{handler="rules"} 0 -http_response_size_bytes{handler="series",quantile="0.5"} NaN -http_response_size_bytes{handler="series",quantile="0.9"} NaN -http_response_size_bytes{handler="series",quantile="0.99"} NaN -http_response_size_bytes_sum{handler="series"} 0 -http_response_size_bytes_count{handler="series"} 0 -http_response_size_bytes{handler="static",quantile="0.5"} 6316 -http_response_size_bytes{handler="static",quantile="0.9"} 6316 -http_response_size_bytes{handler="static",quantile="0.99"} 6316 -http_response_size_bytes_sum{handler="static"} 18948 -http_response_size_bytes_count{handler="static"} 3 -http_response_size_bytes{handler="status",quantile="0.5"} NaN -http_response_size_bytes{handler="status",quantile="0.9"} NaN -http_response_size_bytes{handler="status",quantile="0.99"} NaN -http_response_size_bytes_sum{handler="status"} 0 -http_response_size_bytes_count{handler="status"} 0 -http_response_size_bytes{handler="targets",quantile="0.5"} NaN -http_response_size_bytes{handler="targets",quantile="0.9"} NaN -http_response_size_bytes{handler="targets",quantile="0.99"} NaN -http_response_size_bytes_sum{handler="targets"} 0 -http_response_size_bytes_count{handler="targets"} 0 -http_response_size_bytes{handler="version",quantile="0.5"} NaN -http_response_size_bytes{handler="version",quantile="0.9"} NaN -http_response_size_bytes{handler="version",quantile="0.99"} NaN -http_response_size_bytes_sum{handler="version"} 0 -http_response_size_bytes_count{handler="version"} 0 -prometheus_build_info{branch="",goversion="go1.7.3",revision="",version=""} 1 -prometheus_config_last_reload_success_timestamp_seconds 1.484395547e+09 -prometheus_config_last_reload_successful 1 -prometheus_evaluator_duration_seconds{quantile="0.01"} 1.7890000000000002e-06 -prometheus_evaluator_duration_seconds{quantile="0.05"} 1.7890000000000002e-06 -prometheus_evaluator_duration_seconds{quantile="0.5"} 1.7890000000000002e-06 -prometheus_evaluator_duration_seconds{quantile="0.9"} 1.7890000000000002e-06 -prometheus_evaluator_duration_seconds{quantile="0.99"} 1.7890000000000002e-06 -prometheus_evaluator_duration_seconds_sum 1.7890000000000002e-06 -prometheus_evaluator_duration_seconds_count 1 -prometheus_evaluator_iterations_skipped_total 0 -prometheus_notifications_dropped_total 0 -prometheus_notifications_queue_capacity 10000 -prometheus_notifications_queue_length 0 -prometheus_rule_evaluation_failures_total{rule_type="alerting"} 0 -prometheus_rule_evaluation_failures_total{rule_type="recording"} 0 -prometheus_sd_azure_refresh_duration_seconds{quantile="0.5"} NaN -prometheus_sd_azure_refresh_duration_seconds{quantile="0.9"} NaN -prometheus_sd_azure_refresh_duration_seconds{quantile="0.99"} NaN -prometheus_sd_azure_refresh_duration_seconds_sum 0 -prometheus_sd_azure_refresh_duration_seconds_count 0 -prometheus_sd_azure_refresh_failures_total 0 -prometheus_sd_consul_rpc_duration_seconds{call="service",endpoint="catalog",quantile="0.5"} NaN -prometheus_sd_consul_rpc_duration_seconds{call="service",endpoint="catalog",quantile="0.9"} NaN -prometheus_sd_consul_rpc_duration_seconds{call="service",endpoint="catalog",quantile="0.99"} NaN -prometheus_sd_consul_rpc_duration_seconds_sum{call="service",endpoint="catalog"} 0 -prometheus_sd_consul_rpc_duration_seconds_count{call="service",endpoint="catalog"} 0 -prometheus_sd_consul_rpc_duration_seconds{call="services",endpoint="catalog",quantile="0.5"} NaN -prometheus_sd_consul_rpc_duration_seconds{call="services",endpoint="catalog",quantile="0.9"} NaN -prometheus_sd_consul_rpc_duration_seconds{call="services",endpoint="catalog",quantile="0.99"} NaN -prometheus_sd_consul_rpc_duration_seconds_sum{call="services",endpoint="catalog"} 0 -prometheus_sd_consul_rpc_duration_seconds_count{call="services",endpoint="catalog"} 0 -prometheus_sd_consul_rpc_failures_total 0 -prometheus_sd_dns_lookup_failures_total 0 -prometheus_sd_dns_lookups_total 0 -prometheus_sd_ec2_refresh_duration_seconds{quantile="0.5"} NaN -prometheus_sd_ec2_refresh_duration_seconds{quantile="0.9"} NaN -prometheus_sd_ec2_refresh_duration_seconds{quantile="0.99"} NaN -prometheus_sd_ec2_refresh_duration_seconds_sum 0 -prometheus_sd_ec2_refresh_duration_seconds_count 0 -prometheus_sd_ec2_refresh_failures_total 0 -prometheus_sd_file_read_errors_total 0 -prometheus_sd_file_scan_duration_seconds{quantile="0.5"} NaN -prometheus_sd_file_scan_duration_seconds{quantile="0.9"} NaN -prometheus_sd_file_scan_duration_seconds{quantile="0.99"} NaN -prometheus_sd_file_scan_duration_seconds_sum 0 -prometheus_sd_file_scan_duration_seconds_count 0 -prometheus_sd_gce_refresh_duration{quantile="0.5"} NaN -prometheus_sd_gce_refresh_duration{quantile="0.9"} NaN -prometheus_sd_gce_refresh_duration{quantile="0.99"} NaN -prometheus_sd_gce_refresh_duration_sum 0 -prometheus_sd_gce_refresh_duration_count 0 -prometheus_sd_gce_refresh_failures_total 0 -prometheus_sd_kubernetes_events_total{event="add",role="endpoints"} 0 -prometheus_sd_kubernetes_events_total{event="add",role="node"} 0 -prometheus_sd_kubernetes_events_total{event="add",role="pod"} 0 -prometheus_sd_kubernetes_events_total{event="add",role="service"} 0 -prometheus_sd_kubernetes_events_total{event="delete",role="endpoints"} 0 -prometheus_sd_kubernetes_events_total{event="delete",role="node"} 0 -prometheus_sd_kubernetes_events_total{event="delete",role="pod"} 0 -prometheus_sd_kubernetes_events_total{event="delete",role="service"} 0 -prometheus_sd_kubernetes_events_total{event="update",role="endpoints"} 0 -prometheus_sd_kubernetes_events_total{event="update",role="node"} 0 -prometheus_sd_kubernetes_events_total{event="update",role="pod"} 0 -prometheus_sd_kubernetes_events_total{event="update",role="service"} 0 -prometheus_sd_marathon_refresh_duration_seconds{quantile="0.5"} NaN -prometheus_sd_marathon_refresh_duration_seconds{quantile="0.9"} NaN -prometheus_sd_marathon_refresh_duration_seconds{quantile="0.99"} NaN -prometheus_sd_marathon_refresh_duration_seconds_sum 0 -prometheus_sd_marathon_refresh_duration_seconds_count 0 -prometheus_sd_marathon_refresh_failures_total 0 -prometheus_target_interval_length_seconds{interval="50ms",quantile="0.01"} 0.046182157 -prometheus_target_interval_length_seconds{interval="50ms",quantile="0.05"} 0.047306979000000006 -prometheus_target_interval_length_seconds{interval="50ms",quantile="0.5"} 0.050381782 -prometheus_target_interval_length_seconds{interval="50ms",quantile="0.9"} 0.052614556 -prometheus_target_interval_length_seconds{interval="50ms",quantile="0.99"} 0.054404386000000006 -prometheus_target_interval_length_seconds_sum{interval="50ms"} 34.512091221999995 -prometheus_target_interval_length_seconds_count{interval="50ms"} 685 -prometheus_target_scrape_pool_sync_total{scrape_job="prometheus"} 1 -prometheus_target_skipped_scrapes_total 0 -prometheus_target_sync_length_seconds{scrape_job="prometheus",quantile="0.01"} 0.00020043300000000002 -prometheus_target_sync_length_seconds{scrape_job="prometheus",quantile="0.05"} 0.00020043300000000002 -prometheus_target_sync_length_seconds{scrape_job="prometheus",quantile="0.5"} 0.00020043300000000002 -prometheus_target_sync_length_seconds{scrape_job="prometheus",quantile="0.9"} 0.00020043300000000002 -prometheus_target_sync_length_seconds{scrape_job="prometheus",quantile="0.99"} 0.00020043300000000002 -prometheus_target_sync_length_seconds_sum{scrape_job="prometheus"} 0.00020043300000000002 -prometheus_target_sync_length_seconds_count{scrape_job="prometheus"} 1 -prometheus_treecache_watcher_goroutines 0 -prometheus_treecache_zookeeper_failures_total 0 -# EOF diff --git a/model/textparse/testdata/promtestdata.txt b/model/textparse/testdata/promtestdata.txt deleted file mode 100644 index 174f383e91..0000000000 --- a/model/textparse/testdata/promtestdata.txt +++ /dev/null @@ -1,529 +0,0 @@ -# HELP go_gc_duration_seconds A summary of the GC invocation durations. -# TYPE go_gc_duration_seconds summary -go_gc_duration_seconds{quantile="0"} 4.9351e-05 -go_gc_duration_seconds{quantile="0.25"} 7.424100000000001e-05 -go_gc_duration_seconds{quantile="0.5"} 8.3835e-05 -go_gc_duration_seconds{quantile="0.75"} 0.000106744 -go_gc_duration_seconds{quantile="1"} 0.002072195 -go_gc_duration_seconds_sum 0.012139815 -go_gc_duration_seconds_count 99 -# HELP go_goroutines Number of goroutines that currently exist. -# TYPE go_goroutines gauge -go_goroutines 33 -# HELP go_memstats_alloc_bytes Number of bytes allocated and still in use. -# TYPE go_memstats_alloc_bytes gauge -go_memstats_alloc_bytes 1.7518624e+07 -# HELP go_memstats_alloc_bytes_total Total number of bytes allocated, even if freed. -# TYPE go_memstats_alloc_bytes_total counter -go_memstats_alloc_bytes_total 8.3062296e+08 -# HELP go_memstats_buck_hash_sys_bytes Number of bytes used by the profiling bucket hash table. -# TYPE go_memstats_buck_hash_sys_bytes gauge -go_memstats_buck_hash_sys_bytes 1.494637e+06 -# HELP go_memstats_frees_total Total number of frees. -# TYPE go_memstats_frees_total counter -go_memstats_frees_total 4.65658e+06 -# HELP go_memstats_gc_sys_bytes Number of bytes used for garbage collection system metadata. -# TYPE go_memstats_gc_sys_bytes gauge -go_memstats_gc_sys_bytes 1.107968e+06 -# HELP go_memstats_heap_alloc_bytes Number of heap bytes allocated and still in use. -# TYPE go_memstats_heap_alloc_bytes gauge -go_memstats_heap_alloc_bytes 1.7518624e+07 -# HELP go_memstats_heap_idle_bytes Number of heap bytes waiting to be used. -# TYPE go_memstats_heap_idle_bytes gauge -go_memstats_heap_idle_bytes 6.668288e+06 -# HELP go_memstats_heap_inuse_bytes Number of heap bytes that are in use. -# TYPE go_memstats_heap_inuse_bytes gauge -go_memstats_heap_inuse_bytes 1.8956288e+07 -# HELP go_memstats_heap_objects Number of allocated objects. -# TYPE go_memstats_heap_objects gauge -go_memstats_heap_objects 72755 -# HELP go_memstats_heap_released_bytes_total Total number of heap bytes released to OS. -# TYPE go_memstats_heap_released_bytes_total counter -go_memstats_heap_released_bytes_total 0 -# HELP go_memstats_heap_sys_bytes Number of heap bytes obtained from system. -# TYPE go_memstats_heap_sys_bytes gauge -go_memstats_heap_sys_bytes 2.5624576e+07 -# HELP go_memstats_last_gc_time_seconds Number of seconds since 1970 of last garbage collection. -# TYPE go_memstats_last_gc_time_seconds gauge -go_memstats_last_gc_time_seconds 1.4843955586166437e+09 -# HELP go_memstats_lookups_total Total number of pointer lookups. -# TYPE go_memstats_lookups_total counter -go_memstats_lookups_total 2089 -# HELP go_memstats_mallocs_total Total number of mallocs. -# TYPE go_memstats_mallocs_total counter -go_memstats_mallocs_total 4.729335e+06 -# HELP go_memstats_mcache_inuse_bytes Number of bytes in use by mcache structures. -# TYPE go_memstats_mcache_inuse_bytes gauge -go_memstats_mcache_inuse_bytes 9600 -# HELP go_memstats_mcache_sys_bytes Number of bytes used for mcache structures obtained from system. -# TYPE go_memstats_mcache_sys_bytes gauge -go_memstats_mcache_sys_bytes 16384 -# HELP go_memstats_mspan_inuse_bytes Number of bytes in use by mspan structures. -# TYPE go_memstats_mspan_inuse_bytes gauge -go_memstats_mspan_inuse_bytes 211520 -# HELP go_memstats_mspan_sys_bytes Number of bytes used for mspan structures obtained from system. -# TYPE go_memstats_mspan_sys_bytes gauge -go_memstats_mspan_sys_bytes 245760 -# HELP go_memstats_next_gc_bytes Number of heap bytes when next garbage collection will take place. -# TYPE go_memstats_next_gc_bytes gauge -go_memstats_next_gc_bytes 2.033527e+07 -# HELP go_memstats_other_sys_bytes Number of bytes used for other system allocations. -# TYPE go_memstats_other_sys_bytes gauge -go_memstats_other_sys_bytes 2.077323e+06 -# HELP go_memstats_stack_inuse_bytes Number of bytes in use by the stack allocator. -# TYPE go_memstats_stack_inuse_bytes gauge -go_memstats_stack_inuse_bytes 1.6384e+06 -# HELP go_memstats_stack_sys_bytes Number of bytes obtained from system for stack allocator. -# TYPE go_memstats_stack_sys_bytes gauge -go_memstats_stack_sys_bytes 1.6384e+06 -# HELP go_memstats_sys_bytes Number of bytes obtained by system. Sum of all system allocations. -# TYPE go_memstats_sys_bytes gauge -go_memstats_sys_bytes 3.2205048e+07 -# HELP http_request_duration_microseconds The HTTP request latencies in microseconds. -# TYPE http_request_duration_microseconds summary -http_request_duration_microseconds{handler="alerts",quantile="0.5"} NaN -http_request_duration_microseconds{handler="alerts",quantile="0.9"} NaN -http_request_duration_microseconds{handler="alerts",quantile="0.99"} NaN -http_request_duration_microseconds_sum{handler="alerts"} 0 -http_request_duration_microseconds_count{handler="alerts"} 0 -http_request_duration_microseconds{handler="config",quantile="0.5"} NaN -http_request_duration_microseconds{handler="config",quantile="0.9"} NaN -http_request_duration_microseconds{handler="config",quantile="0.99"} NaN -http_request_duration_microseconds_sum{handler="config"} 0 -http_request_duration_microseconds_count{handler="config"} 0 -http_request_duration_microseconds{handler="consoles",quantile="0.5"} NaN -http_request_duration_microseconds{handler="consoles",quantile="0.9"} NaN -http_request_duration_microseconds{handler="consoles",quantile="0.99"} NaN -http_request_duration_microseconds_sum{handler="consoles"} 0 -http_request_duration_microseconds_count{handler="consoles"} 0 -http_request_duration_microseconds{handler="drop_series",quantile="0.5"} NaN -http_request_duration_microseconds{handler="drop_series",quantile="0.9"} NaN -http_request_duration_microseconds{handler="drop_series",quantile="0.99"} NaN -http_request_duration_microseconds_sum{handler="drop_series"} 0 -http_request_duration_microseconds_count{handler="drop_series"} 0 -http_request_duration_microseconds{handler="federate",quantile="0.5"} NaN -http_request_duration_microseconds{handler="federate",quantile="0.9"} NaN -http_request_duration_microseconds{handler="federate",quantile="0.99"} NaN -http_request_duration_microseconds_sum{handler="federate"} 0 -http_request_duration_microseconds_count{handler="federate"} 0 -http_request_duration_microseconds{handler="flags",quantile="0.5"} NaN -http_request_duration_microseconds{handler="flags",quantile="0.9"} NaN -http_request_duration_microseconds{handler="flags",quantile="0.99"} NaN -http_request_duration_microseconds_sum{handler="flags"} 0 -http_request_duration_microseconds_count{handler="flags"} 0 -http_request_duration_microseconds{handler="graph",quantile="0.5"} 771.655 -http_request_duration_microseconds{handler="graph",quantile="0.9"} 1761.823 -http_request_duration_microseconds{handler="graph",quantile="0.99"} 1761.823 -http_request_duration_microseconds_sum{handler="graph"} 5803.93 -http_request_duration_microseconds_count{handler="graph"} 3 -http_request_duration_microseconds{handler="heap",quantile="0.5"} NaN -http_request_duration_microseconds{handler="heap",quantile="0.9"} NaN -http_request_duration_microseconds{handler="heap",quantile="0.99"} NaN -http_request_duration_microseconds_sum{handler="heap"} 0 -http_request_duration_microseconds_count{handler="heap"} 0 -http_request_duration_microseconds{handler="label_values",quantile="0.5"} 325.401 -http_request_duration_microseconds{handler="label_values",quantile="0.9"} 414.708 -http_request_duration_microseconds{handler="label_values",quantile="0.99"} 414.708 -http_request_duration_microseconds_sum{handler="label_values"} 3995.574 -http_request_duration_microseconds_count{handler="label_values"} 3 -http_request_duration_microseconds{handler="options",quantile="0.5"} NaN -http_request_duration_microseconds{handler="options",quantile="0.9"} NaN -http_request_duration_microseconds{handler="options",quantile="0.99"} NaN -http_request_duration_microseconds_sum{handler="options"} 0 -http_request_duration_microseconds_count{handler="options"} 0 -http_request_duration_microseconds{handler="prometheus",quantile="0.5"} 1351.859 -http_request_duration_microseconds{handler="prometheus",quantile="0.9"} 1714.035 -http_request_duration_microseconds{handler="prometheus",quantile="0.99"} 2833.523 -http_request_duration_microseconds_sum{handler="prometheus"} 661851.54 -http_request_duration_microseconds_count{handler="prometheus"} 462 -http_request_duration_microseconds{handler="query",quantile="0.5"} 3885.448 -http_request_duration_microseconds{handler="query",quantile="0.9"} 4390.558 -http_request_duration_microseconds{handler="query",quantile="0.99"} 4390.558 -http_request_duration_microseconds_sum{handler="query"} 26074.11 -http_request_duration_microseconds_count{handler="query"} 6 -http_request_duration_microseconds{handler="query_range",quantile="0.5"} NaN -http_request_duration_microseconds{handler="query_range",quantile="0.9"} NaN -http_request_duration_microseconds{handler="query_range",quantile="0.99"} NaN -http_request_duration_microseconds_sum{handler="query_range"} 0 -http_request_duration_microseconds_count{handler="query_range"} 0 -http_request_duration_microseconds{handler="rules",quantile="0.5"} NaN -http_request_duration_microseconds{handler="rules",quantile="0.9"} NaN -http_request_duration_microseconds{handler="rules",quantile="0.99"} NaN -http_request_duration_microseconds_sum{handler="rules"} 0 -http_request_duration_microseconds_count{handler="rules"} 0 -http_request_duration_microseconds{handler="series",quantile="0.5"} NaN -http_request_duration_microseconds{handler="series",quantile="0.9"} NaN -http_request_duration_microseconds{handler="series",quantile="0.99"} NaN -http_request_duration_microseconds_sum{handler="series"} 0 -http_request_duration_microseconds_count{handler="series"} 0 -http_request_duration_microseconds{handler="static",quantile="0.5"} 212.311 -http_request_duration_microseconds{handler="static",quantile="0.9"} 265.174 -http_request_duration_microseconds{handler="static",quantile="0.99"} 265.174 -http_request_duration_microseconds_sum{handler="static"} 6458.621 -http_request_duration_microseconds_count{handler="static"} 3 -http_request_duration_microseconds{handler="status",quantile="0.5"} NaN -http_request_duration_microseconds{handler="status",quantile="0.9"} NaN -http_request_duration_microseconds{handler="status",quantile="0.99"} NaN -http_request_duration_microseconds_sum{handler="status"} 0 -http_request_duration_microseconds_count{handler="status"} 0 -http_request_duration_microseconds{handler="targets",quantile="0.5"} NaN -http_request_duration_microseconds{handler="targets",quantile="0.9"} NaN -http_request_duration_microseconds{handler="targets",quantile="0.99"} NaN -http_request_duration_microseconds_sum{handler="targets"} 0 -http_request_duration_microseconds_count{handler="targets"} 0 -http_request_duration_microseconds{handler="version",quantile="0.5"} NaN -http_request_duration_microseconds{handler="version",quantile="0.9"} NaN -http_request_duration_microseconds{handler="version",quantile="0.99"} NaN -http_request_duration_microseconds_sum{handler="version"} 0 -http_request_duration_microseconds_count{handler="version"} 0 -# HELP http_request_size_bytes The HTTP request sizes in bytes. -# TYPE http_request_size_bytes summary -http_request_size_bytes{handler="alerts",quantile="0.5"} NaN -http_request_size_bytes{handler="alerts",quantile="0.9"} NaN -http_request_size_bytes{handler="alerts",quantile="0.99"} NaN -http_request_size_bytes_sum{handler="alerts"} 0 -http_request_size_bytes_count{handler="alerts"} 0 -http_request_size_bytes{handler="config",quantile="0.5"} NaN -http_request_size_bytes{handler="config",quantile="0.9"} NaN -http_request_size_bytes{handler="config",quantile="0.99"} NaN -http_request_size_bytes_sum{handler="config"} 0 -http_request_size_bytes_count{handler="config"} 0 -http_request_size_bytes{handler="consoles",quantile="0.5"} NaN -http_request_size_bytes{handler="consoles",quantile="0.9"} NaN -http_request_size_bytes{handler="consoles",quantile="0.99"} NaN -http_request_size_bytes_sum{handler="consoles"} 0 -http_request_size_bytes_count{handler="consoles"} 0 -http_request_size_bytes{handler="drop_series",quantile="0.5"} NaN -http_request_size_bytes{handler="drop_series",quantile="0.9"} NaN -http_request_size_bytes{handler="drop_series",quantile="0.99"} NaN -http_request_size_bytes_sum{handler="drop_series"} 0 -http_request_size_bytes_count{handler="drop_series"} 0 -http_request_size_bytes{handler="federate",quantile="0.5"} NaN -http_request_size_bytes{handler="federate",quantile="0.9"} NaN -http_request_size_bytes{handler="federate",quantile="0.99"} NaN -http_request_size_bytes_sum{handler="federate"} 0 -http_request_size_bytes_count{handler="federate"} 0 -http_request_size_bytes{handler="flags",quantile="0.5"} NaN -http_request_size_bytes{handler="flags",quantile="0.9"} NaN -http_request_size_bytes{handler="flags",quantile="0.99"} NaN -http_request_size_bytes_sum{handler="flags"} 0 -http_request_size_bytes_count{handler="flags"} 0 -http_request_size_bytes{handler="graph",quantile="0.5"} 367 -http_request_size_bytes{handler="graph",quantile="0.9"} 389 -http_request_size_bytes{handler="graph",quantile="0.99"} 389 -http_request_size_bytes_sum{handler="graph"} 1145 -http_request_size_bytes_count{handler="graph"} 3 -http_request_size_bytes{handler="heap",quantile="0.5"} NaN -http_request_size_bytes{handler="heap",quantile="0.9"} NaN -http_request_size_bytes{handler="heap",quantile="0.99"} NaN -http_request_size_bytes_sum{handler="heap"} 0 -http_request_size_bytes_count{handler="heap"} 0 -http_request_size_bytes{handler="label_values",quantile="0.5"} 416 -http_request_size_bytes{handler="label_values",quantile="0.9"} 416 -http_request_size_bytes{handler="label_values",quantile="0.99"} 416 -http_request_size_bytes_sum{handler="label_values"} 1248 -http_request_size_bytes_count{handler="label_values"} 3 -http_request_size_bytes{handler="options",quantile="0.5"} NaN -http_request_size_bytes{handler="options",quantile="0.9"} NaN -http_request_size_bytes{handler="options",quantile="0.99"} NaN -http_request_size_bytes_sum{handler="options"} 0 -http_request_size_bytes_count{handler="options"} 0 -http_request_size_bytes{handler="prometheus",quantile="0.5"} 238 -http_request_size_bytes{handler="prometheus",quantile="0.9"} 238 -http_request_size_bytes{handler="prometheus",quantile="0.99"} 238 -http_request_size_bytes_sum{handler="prometheus"} 109956 -http_request_size_bytes_count{handler="prometheus"} 462 -http_request_size_bytes{handler="query",quantile="0.5"} 531 -http_request_size_bytes{handler="query",quantile="0.9"} 531 -http_request_size_bytes{handler="query",quantile="0.99"} 531 -http_request_size_bytes_sum{handler="query"} 3186 -http_request_size_bytes_count{handler="query"} 6 -http_request_size_bytes{handler="query_range",quantile="0.5"} NaN -http_request_size_bytes{handler="query_range",quantile="0.9"} NaN -http_request_size_bytes{handler="query_range",quantile="0.99"} NaN -http_request_size_bytes_sum{handler="query_range"} 0 -http_request_size_bytes_count{handler="query_range"} 0 -http_request_size_bytes{handler="rules",quantile="0.5"} NaN -http_request_size_bytes{handler="rules",quantile="0.9"} NaN -http_request_size_bytes{handler="rules",quantile="0.99"} NaN -http_request_size_bytes_sum{handler="rules"} 0 -http_request_size_bytes_count{handler="rules"} 0 -http_request_size_bytes{handler="series",quantile="0.5"} NaN -http_request_size_bytes{handler="series",quantile="0.9"} NaN -http_request_size_bytes{handler="series",quantile="0.99"} NaN -http_request_size_bytes_sum{handler="series"} 0 -http_request_size_bytes_count{handler="series"} 0 -http_request_size_bytes{handler="static",quantile="0.5"} 379 -http_request_size_bytes{handler="static",quantile="0.9"} 379 -http_request_size_bytes{handler="static",quantile="0.99"} 379 -http_request_size_bytes_sum{handler="static"} 1137 -http_request_size_bytes_count{handler="static"} 3 -http_request_size_bytes{handler="status",quantile="0.5"} NaN -http_request_size_bytes{handler="status",quantile="0.9"} NaN -http_request_size_bytes{handler="status",quantile="0.99"} NaN -http_request_size_bytes_sum{handler="status"} 0 -http_request_size_bytes_count{handler="status"} 0 -http_request_size_bytes{handler="targets",quantile="0.5"} NaN -http_request_size_bytes{handler="targets",quantile="0.9"} NaN -http_request_size_bytes{handler="targets",quantile="0.99"} NaN -http_request_size_bytes_sum{handler="targets"} 0 -http_request_size_bytes_count{handler="targets"} 0 -http_request_size_bytes{handler="version",quantile="0.5"} NaN -http_request_size_bytes{handler="version",quantile="0.9"} NaN -http_request_size_bytes{handler="version",quantile="0.99"} NaN -http_request_size_bytes_sum{handler="version"} 0 -http_request_size_bytes_count{handler="version"} 0 -# HELP http_requests_total Total number of HTTP requests made. -# TYPE http_requests_total counter -http_requests_total{code="200",handler="graph",method="get"} 3 -http_requests_total{code="200",handler="label_values",method="get"} 3 -http_requests_total{code="200",handler="prometheus",method="get"} 462 -http_requests_total{code="200",handler="query",method="get"} 6 -http_requests_total{code="200",handler="static",method="get"} 3 -# HELP http_response_size_bytes The HTTP response sizes in bytes. -# TYPE http_response_size_bytes summary -http_response_size_bytes{handler="alerts",quantile="0.5"} NaN -http_response_size_bytes{handler="alerts",quantile="0.9"} NaN -http_response_size_bytes{handler="alerts",quantile="0.99"} NaN -http_response_size_bytes_sum{handler="alerts"} 0 -http_response_size_bytes_count{handler="alerts"} 0 -http_response_size_bytes{handler="config",quantile="0.5"} NaN -http_response_size_bytes{handler="config",quantile="0.9"} NaN -http_response_size_bytes{handler="config",quantile="0.99"} NaN -http_response_size_bytes_sum{handler="config"} 0 -http_response_size_bytes_count{handler="config"} 0 -http_response_size_bytes{handler="consoles",quantile="0.5"} NaN -http_response_size_bytes{handler="consoles",quantile="0.9"} NaN -http_response_size_bytes{handler="consoles",quantile="0.99"} NaN -http_response_size_bytes_sum{handler="consoles"} 0 -http_response_size_bytes_count{handler="consoles"} 0 -http_response_size_bytes{handler="drop_series",quantile="0.5"} NaN -http_response_size_bytes{handler="drop_series",quantile="0.9"} NaN -http_response_size_bytes{handler="drop_series",quantile="0.99"} NaN -http_response_size_bytes_sum{handler="drop_series"} 0 -http_response_size_bytes_count{handler="drop_series"} 0 -http_response_size_bytes{handler="federate",quantile="0.5"} NaN -http_response_size_bytes{handler="federate",quantile="0.9"} NaN -http_response_size_bytes{handler="federate",quantile="0.99"} NaN -http_response_size_bytes_sum{handler="federate"} 0 -http_response_size_bytes_count{handler="federate"} 0 -http_response_size_bytes{handler="flags",quantile="0.5"} NaN -http_response_size_bytes{handler="flags",quantile="0.9"} NaN -http_response_size_bytes{handler="flags",quantile="0.99"} NaN -http_response_size_bytes_sum{handler="flags"} 0 -http_response_size_bytes_count{handler="flags"} 0 -http_response_size_bytes{handler="graph",quantile="0.5"} 3619 -http_response_size_bytes{handler="graph",quantile="0.9"} 3619 -http_response_size_bytes{handler="graph",quantile="0.99"} 3619 -http_response_size_bytes_sum{handler="graph"} 10857 -http_response_size_bytes_count{handler="graph"} 3 -http_response_size_bytes{handler="heap",quantile="0.5"} NaN -http_response_size_bytes{handler="heap",quantile="0.9"} NaN -http_response_size_bytes{handler="heap",quantile="0.99"} NaN -http_response_size_bytes_sum{handler="heap"} 0 -http_response_size_bytes_count{handler="heap"} 0 -http_response_size_bytes{handler="label_values",quantile="0.5"} 642 -http_response_size_bytes{handler="label_values",quantile="0.9"} 642 -http_response_size_bytes{handler="label_values",quantile="0.99"} 642 -http_response_size_bytes_sum{handler="label_values"} 1926 -http_response_size_bytes_count{handler="label_values"} 3 -http_response_size_bytes{handler="options",quantile="0.5"} NaN -http_response_size_bytes{handler="options",quantile="0.9"} NaN -http_response_size_bytes{handler="options",quantile="0.99"} NaN -http_response_size_bytes_sum{handler="options"} 0 -http_response_size_bytes_count{handler="options"} 0 -http_response_size_bytes{handler="prometheus",quantile="0.5"} 3033 -http_response_size_bytes{handler="prometheus",quantile="0.9"} 3123 -http_response_size_bytes{handler="prometheus",quantile="0.99"} 3128 -http_response_size_bytes_sum{handler="prometheus"} 1.374097e+06 -http_response_size_bytes_count{handler="prometheus"} 462 -http_response_size_bytes{handler="query",quantile="0.5"} 776 -http_response_size_bytes{handler="query",quantile="0.9"} 781 -http_response_size_bytes{handler="query",quantile="0.99"} 781 -http_response_size_bytes_sum{handler="query"} 4656 -http_response_size_bytes_count{handler="query"} 6 -http_response_size_bytes{handler="query_range",quantile="0.5"} NaN -http_response_size_bytes{handler="query_range",quantile="0.9"} NaN -http_response_size_bytes{handler="query_range",quantile="0.99"} NaN -http_response_size_bytes_sum{handler="query_range"} 0 -http_response_size_bytes_count{handler="query_range"} 0 -http_response_size_bytes{handler="rules",quantile="0.5"} NaN -http_response_size_bytes{handler="rules",quantile="0.9"} NaN -http_response_size_bytes{handler="rules",quantile="0.99"} NaN -http_response_size_bytes_sum{handler="rules"} 0 -http_response_size_bytes_count{handler="rules"} 0 -http_response_size_bytes{handler="series",quantile="0.5"} NaN -http_response_size_bytes{handler="series",quantile="0.9"} NaN -http_response_size_bytes{handler="series",quantile="0.99"} NaN -http_response_size_bytes_sum{handler="series"} 0 -http_response_size_bytes_count{handler="series"} 0 -http_response_size_bytes{handler="static",quantile="0.5"} 6316 -http_response_size_bytes{handler="static",quantile="0.9"} 6316 -http_response_size_bytes{handler="static",quantile="0.99"} 6316 -http_response_size_bytes_sum{handler="static"} 18948 -http_response_size_bytes_count{handler="static"} 3 -http_response_size_bytes{handler="status",quantile="0.5"} NaN -http_response_size_bytes{handler="status",quantile="0.9"} NaN -http_response_size_bytes{handler="status",quantile="0.99"} NaN -http_response_size_bytes_sum{handler="status"} 0 -http_response_size_bytes_count{handler="status"} 0 -http_response_size_bytes{handler="targets",quantile="0.5"} NaN -http_response_size_bytes{handler="targets",quantile="0.9"} NaN -http_response_size_bytes{handler="targets",quantile="0.99"} NaN -http_response_size_bytes_sum{handler="targets"} 0 -http_response_size_bytes_count{handler="targets"} 0 -http_response_size_bytes{handler="version",quantile="0.5"} NaN -http_response_size_bytes{handler="version",quantile="0.9"} NaN -http_response_size_bytes{handler="version",quantile="0.99"} NaN -http_response_size_bytes_sum{handler="version"} 0 -http_response_size_bytes_count{handler="version"} 0 -# HELP prometheus_build_info A metric with a constant '1' value labeled by version, revision, branch, and goversion from which prometheus was built. -# TYPE prometheus_build_info gauge -prometheus_build_info{branch="",goversion="go1.7.3",revision="",version=""} 1 -# HELP prometheus_config_last_reload_success_timestamp_seconds Timestamp of the last successful configuration reload. -# TYPE prometheus_config_last_reload_success_timestamp_seconds gauge -prometheus_config_last_reload_success_timestamp_seconds 1.484395547e+09 -# HELP prometheus_config_last_reload_successful Whether the last configuration reload attempt was successful. -# TYPE prometheus_config_last_reload_successful gauge -prometheus_config_last_reload_successful 1 -# HELP prometheus_evaluator_duration_seconds The duration of rule group evaluations. -# TYPE prometheus_evaluator_duration_seconds summary -prometheus_evaluator_duration_seconds{quantile="0.01"} 1.7890000000000002e-06 -prometheus_evaluator_duration_seconds{quantile="0.05"} 1.7890000000000002e-06 -prometheus_evaluator_duration_seconds{quantile="0.5"} 1.7890000000000002e-06 -prometheus_evaluator_duration_seconds{quantile="0.9"} 1.7890000000000002e-06 -prometheus_evaluator_duration_seconds{quantile="0.99"} 1.7890000000000002e-06 -prometheus_evaluator_duration_seconds_sum 1.7890000000000002e-06 -prometheus_evaluator_duration_seconds_count 1 -# HELP prometheus_evaluator_iterations_skipped_total The total number of rule group evaluations skipped due to throttled metric storage. -# TYPE prometheus_evaluator_iterations_skipped_total counter -prometheus_evaluator_iterations_skipped_total 0 -# HELP prometheus_notifications_dropped_total Total number of alerts dropped due to alert manager missing in configuration. -# TYPE prometheus_notifications_dropped_total counter -prometheus_notifications_dropped_total 0 -# HELP prometheus_notifications_queue_capacity The capacity of the alert notifications queue. -# TYPE prometheus_notifications_queue_capacity gauge -prometheus_notifications_queue_capacity 10000 -# HELP prometheus_notifications_queue_length The number of alert notifications in the queue. -# TYPE prometheus_notifications_queue_length gauge -prometheus_notifications_queue_length 0 -# HELP prometheus_rule_evaluation_failures_total The total number of rule evaluation failures. -# TYPE prometheus_rule_evaluation_failures_total counter -prometheus_rule_evaluation_failures_total{rule_type="alerting"} 0 -prometheus_rule_evaluation_failures_total{rule_type="recording"} 0 -# HELP prometheus_sd_azure_refresh_duration_seconds The duration of a Azure-SD refresh in seconds. -# TYPE prometheus_sd_azure_refresh_duration_seconds summary -prometheus_sd_azure_refresh_duration_seconds{quantile="0.5"} NaN -prometheus_sd_azure_refresh_duration_seconds{quantile="0.9"} NaN -prometheus_sd_azure_refresh_duration_seconds{quantile="0.99"} NaN -prometheus_sd_azure_refresh_duration_seconds_sum 0 -prometheus_sd_azure_refresh_duration_seconds_count 0 -# HELP prometheus_sd_azure_refresh_failures_total Number of Azure-SD refresh failures. -# TYPE prometheus_sd_azure_refresh_failures_total counter -prometheus_sd_azure_refresh_failures_total 0 -# HELP prometheus_sd_consul_rpc_duration_seconds The duration of a Consul RPC call in seconds. -# TYPE prometheus_sd_consul_rpc_duration_seconds summary -prometheus_sd_consul_rpc_duration_seconds{call="service",endpoint="catalog",quantile="0.5"} NaN -prometheus_sd_consul_rpc_duration_seconds{call="service",endpoint="catalog",quantile="0.9"} NaN -prometheus_sd_consul_rpc_duration_seconds{call="service",endpoint="catalog",quantile="0.99"} NaN -prometheus_sd_consul_rpc_duration_seconds_sum{call="service",endpoint="catalog"} 0 -prometheus_sd_consul_rpc_duration_seconds_count{call="service",endpoint="catalog"} 0 -prometheus_sd_consul_rpc_duration_seconds{call="services",endpoint="catalog",quantile="0.5"} NaN -prometheus_sd_consul_rpc_duration_seconds{call="services",endpoint="catalog",quantile="0.9"} NaN -prometheus_sd_consul_rpc_duration_seconds{call="services",endpoint="catalog",quantile="0.99"} NaN -prometheus_sd_consul_rpc_duration_seconds_sum{call="services",endpoint="catalog"} 0 -prometheus_sd_consul_rpc_duration_seconds_count{call="services",endpoint="catalog"} 0 -# HELP prometheus_sd_consul_rpc_failures_total The number of Consul RPC call failures. -# TYPE prometheus_sd_consul_rpc_failures_total counter -prometheus_sd_consul_rpc_failures_total 0 -# HELP prometheus_sd_dns_lookup_failures_total The number of DNS-SD lookup failures. -# TYPE prometheus_sd_dns_lookup_failures_total counter -prometheus_sd_dns_lookup_failures_total 0 -# HELP prometheus_sd_dns_lookups_total The number of DNS-SD lookups. -# TYPE prometheus_sd_dns_lookups_total counter -prometheus_sd_dns_lookups_total 0 -# HELP prometheus_sd_ec2_refresh_duration_seconds The duration of a EC2-SD refresh in seconds. -# TYPE prometheus_sd_ec2_refresh_duration_seconds summary -prometheus_sd_ec2_refresh_duration_seconds{quantile="0.5"} NaN -prometheus_sd_ec2_refresh_duration_seconds{quantile="0.9"} NaN -prometheus_sd_ec2_refresh_duration_seconds{quantile="0.99"} NaN -prometheus_sd_ec2_refresh_duration_seconds_sum 0 -prometheus_sd_ec2_refresh_duration_seconds_count 0 -# HELP prometheus_sd_ec2_refresh_failures_total The number of EC2-SD scrape failures. -# TYPE prometheus_sd_ec2_refresh_failures_total counter -prometheus_sd_ec2_refresh_failures_total 0 -# HELP prometheus_sd_file_read_errors_total The number of File-SD read errors. -# TYPE prometheus_sd_file_read_errors_total counter -prometheus_sd_file_read_errors_total 0 -# HELP prometheus_sd_file_scan_duration_seconds The duration of the File-SD scan in seconds. -# TYPE prometheus_sd_file_scan_duration_seconds summary -prometheus_sd_file_scan_duration_seconds{quantile="0.5"} NaN -prometheus_sd_file_scan_duration_seconds{quantile="0.9"} NaN -prometheus_sd_file_scan_duration_seconds{quantile="0.99"} NaN -prometheus_sd_file_scan_duration_seconds_sum 0 -prometheus_sd_file_scan_duration_seconds_count 0 -# HELP prometheus_sd_gce_refresh_duration The duration of a GCE-SD refresh in seconds. -# TYPE prometheus_sd_gce_refresh_duration summary -prometheus_sd_gce_refresh_duration{quantile="0.5"} NaN -prometheus_sd_gce_refresh_duration{quantile="0.9"} NaN -prometheus_sd_gce_refresh_duration{quantile="0.99"} NaN -prometheus_sd_gce_refresh_duration_sum 0 -prometheus_sd_gce_refresh_duration_count 0 -# HELP prometheus_sd_gce_refresh_failures_total The number of GCE-SD refresh failures. -# TYPE prometheus_sd_gce_refresh_failures_total counter -prometheus_sd_gce_refresh_failures_total 0 -# HELP prometheus_sd_kubernetes_events_total The number of Kubernetes events handled. -# TYPE prometheus_sd_kubernetes_events_total counter -prometheus_sd_kubernetes_events_total{event="add",role="endpoints"} 0 -prometheus_sd_kubernetes_events_total{event="add",role="node"} 0 -prometheus_sd_kubernetes_events_total{event="add",role="pod"} 0 -prometheus_sd_kubernetes_events_total{event="add",role="service"} 0 -prometheus_sd_kubernetes_events_total{event="delete",role="endpoints"} 0 -prometheus_sd_kubernetes_events_total{event="delete",role="node"} 0 -prometheus_sd_kubernetes_events_total{event="delete",role="pod"} 0 -prometheus_sd_kubernetes_events_total{event="delete",role="service"} 0 -prometheus_sd_kubernetes_events_total{event="update",role="endpoints"} 0 -prometheus_sd_kubernetes_events_total{event="update",role="node"} 0 -prometheus_sd_kubernetes_events_total{event="update",role="pod"} 0 -prometheus_sd_kubernetes_events_total{event="update",role="service"} 0 -# HELP prometheus_sd_marathon_refresh_duration_seconds The duration of a Marathon-SD refresh in seconds. -# TYPE prometheus_sd_marathon_refresh_duration_seconds summary -prometheus_sd_marathon_refresh_duration_seconds{quantile="0.5"} NaN -prometheus_sd_marathon_refresh_duration_seconds{quantile="0.9"} NaN -prometheus_sd_marathon_refresh_duration_seconds{quantile="0.99"} NaN -prometheus_sd_marathon_refresh_duration_seconds_sum 0 -prometheus_sd_marathon_refresh_duration_seconds_count 0 -# HELP prometheus_sd_marathon_refresh_failures_total The number of Marathon-SD refresh failures. -# TYPE prometheus_sd_marathon_refresh_failures_total counter -prometheus_sd_marathon_refresh_failures_total 0 -# HELP prometheus_target_interval_length_seconds Actual intervals between scrapes. -# TYPE prometheus_target_interval_length_seconds summary -prometheus_target_interval_length_seconds{interval="50ms",quantile="0.01"} 0.046182157 -prometheus_target_interval_length_seconds{interval="50ms",quantile="0.05"} 0.047306979000000006 -prometheus_target_interval_length_seconds{interval="50ms",quantile="0.5"} 0.050381782 -prometheus_target_interval_length_seconds{interval="50ms",quantile="0.9"} 0.052614556 -prometheus_target_interval_length_seconds{interval="50ms",quantile="0.99"} 0.054404386000000006 -prometheus_target_interval_length_seconds_sum{interval="50ms"} 34.512091221999995 -prometheus_target_interval_length_seconds_count{interval="50ms"} 685 -# HELP prometheus_target_scrape_pool_sync_total Total number of syncs that were executed on a scrape pool. -# TYPE prometheus_target_scrape_pool_sync_total counter -prometheus_target_scrape_pool_sync_total{scrape_job="prometheus"} 1 -# HELP prometheus_target_skipped_scrapes_total Total number of scrapes that were skipped because the metric storage was throttled. -# TYPE prometheus_target_skipped_scrapes_total counter -prometheus_target_skipped_scrapes_total 0 -# HELP prometheus_target_sync_length_seconds Actual interval to sync the scrape pool. -# TYPE prometheus_target_sync_length_seconds summary -prometheus_target_sync_length_seconds{scrape_job="prometheus",quantile="0.01"} 0.00020043300000000002 -prometheus_target_sync_length_seconds{scrape_job="prometheus",quantile="0.05"} 0.00020043300000000002 -prometheus_target_sync_length_seconds{scrape_job="prometheus",quantile="0.5"} 0.00020043300000000002 -prometheus_target_sync_length_seconds{scrape_job="prometheus",quantile="0.9"} 0.00020043300000000002 -prometheus_target_sync_length_seconds{scrape_job="prometheus",quantile="0.99"} 0.00020043300000000002 -prometheus_target_sync_length_seconds_sum{scrape_job="prometheus"} 0.00020043300000000002 -prometheus_target_sync_length_seconds_count{scrape_job="prometheus"} 1 -# HELP prometheus_treecache_watcher_goroutines The current number of watcher goroutines. -# TYPE prometheus_treecache_watcher_goroutines gauge -prometheus_treecache_watcher_goroutines 0 -# HELP prometheus_treecache_zookeeper_failures_total The total number of ZooKeeper failures. -# TYPE prometheus_treecache_zookeeper_failures_total counter -prometheus_treecache_zookeeper_failures_total 0 -# EOF diff --git a/promql/engine.go b/promql/engine.go index 75a0e7dcaf..a082baa261 100644 --- a/promql/engine.go +++ b/promql/engine.go @@ -2495,6 +2495,11 @@ func (ev *evaluator) matrixIterSlice( } } + if mint == maxt { + // Empty range: return the empty slices. + return floats, histograms + } + soughtValueType := it.Seek(maxt) if soughtValueType == chunkenc.ValNone { if it.Err() != nil { diff --git a/promql/engine_test.go b/promql/engine_test.go index 7cf0474246..2361c4fda2 100644 --- a/promql/engine_test.go +++ b/promql/engine_test.go @@ -1900,6 +1900,15 @@ func TestSubquerySelector(t *testing.T) { }, Start: time.Unix(35, 0), }, + { + Query: "metric[0:10s]", + Result: promql.Result{ + nil, + promql.Matrix{}, + nil, + }, + Start: time.Unix(10, 0), + }, }, }, { @@ -3199,6 +3208,7 @@ func TestInstantQueryWithRangeVectorSelector(t *testing.T) { load 1m some_metric{env="1"} 0+1x4 some_metric{env="2"} 0+2x4 + some_metric{env="3"} {{count:0}}+{{count:1}}x4 some_metric_with_stale_marker 0 1 stale 3 `) t.Cleanup(func() { require.NoError(t, storage.Close()) }) @@ -3226,6 +3236,13 @@ func TestInstantQueryWithRangeVectorSelector(t *testing.T) { {T: timestamp.FromTime(baseT.Add(2 * time.Minute)), F: 4}, }, }, + { + Metric: labels.FromStrings("__name__", "some_metric", "env", "3"), + Histograms: []promql.HPoint{ + {T: timestamp.FromTime(baseT.Add(time.Minute)), H: &histogram.FloatHistogram{Count: 1, CounterResetHint: histogram.NotCounterReset}}, + {T: timestamp.FromTime(baseT.Add(2 * time.Minute)), H: &histogram.FloatHistogram{Count: 2, CounterResetHint: histogram.NotCounterReset}}, + }, + }, }, }, "matches no series": { @@ -3251,6 +3268,11 @@ func TestInstantQueryWithRangeVectorSelector(t *testing.T) { }, }, }, + "matches series but range is 0": { + expr: "some_metric[0]", + ts: baseT.Add(2 * time.Minute), + expected: promql.Matrix{}, + }, } for name, testCase := range testCases { diff --git a/promql/functions.go b/promql/functions.go index 938eefdf00..1cb8b2af2d 100644 --- a/promql/functions.go +++ b/promql/functions.go @@ -187,35 +187,48 @@ func extrapolatedRate(vals []parser.Value, args parser.Expressions, enh *EvalNod // not a histogram, and a warning wrapped in an annotation in that case. // Otherwise, it returns the calculated histogram and an empty annotation. func histogramRate(points []HPoint, isCounter bool, metricName string, pos posrange.PositionRange) (*histogram.FloatHistogram, annotations.Annotations) { - prev := points[0].H - usingCustomBuckets := prev.UsesCustomBuckets() - last := points[len(points)-1].H + var ( + prev = points[0].H + usingCustomBuckets = prev.UsesCustomBuckets() + last = points[len(points)-1].H + annos annotations.Annotations + ) + if last == nil { - return nil, annotations.New().Add(annotations.NewMixedFloatsHistogramsWarning(metricName, pos)) + return nil, annos.Add(annotations.NewMixedFloatsHistogramsWarning(metricName, pos)) } - minSchema := prev.Schema - if last.Schema < minSchema { - minSchema = last.Schema + // We check for gauge type histograms in the loop below, but the loop + // below does not run on the first and last point, so check the first + // and last point now. + if isCounter && (prev.CounterResetHint == histogram.GaugeType || last.CounterResetHint == histogram.GaugeType) { + annos.Add(annotations.NewNativeHistogramNotCounterWarning(metricName, pos)) + } + + // Null out the 1st sample if there is a counter reset between the 1st + // and 2nd. In this case, we want to ignore any incompatibility in the + // bucket layout of the 1st sample because we do not need to look at it. + if isCounter && len(points) > 1 { + second := points[1].H + if second != nil && second.DetectReset(prev) { + prev = &histogram.FloatHistogram{} + prev.Schema = second.Schema + prev.CustomValues = second.CustomValues + usingCustomBuckets = second.UsesCustomBuckets() + } } if last.UsesCustomBuckets() != usingCustomBuckets { - return nil, annotations.New().Add(annotations.NewMixedExponentialCustomHistogramsWarning(metricName, pos)) - } - - var annos annotations.Annotations - - // We check for gauge type histograms in the loop below, but the loop below does not run on the first and last point, - // so check the first and last point now. - if isCounter && (prev.CounterResetHint == histogram.GaugeType || last.CounterResetHint == histogram.GaugeType) { - annos.Add(annotations.NewNativeHistogramNotCounterWarning(metricName, pos)) + return nil, annos.Add(annotations.NewMixedExponentialCustomHistogramsWarning(metricName, pos)) } // First iteration to find out two things: // - What's the smallest relevant schema? // - Are all data points histograms? - // TODO(beorn7): Find a way to check that earlier, e.g. by handing in a - // []FloatPoint and a []HistogramPoint separately. + minSchema := prev.Schema + if last.Schema < minSchema { + minSchema = last.Schema + } for _, currPoint := range points[1 : len(points)-1] { curr := currPoint.H if curr == nil { @@ -473,11 +486,22 @@ func funcDoubleExponentialSmoothing(vals []parser.Value, args parser.Expressions return append(enh.Out, Sample{F: s1}), nil } +// filterFloats filters out histogram samples from the vector in-place. +func filterFloats(v Vector) Vector { + floats := v[:0] + for _, s := range v { + if s.H == nil { + floats = append(floats, s) + } + } + return floats +} + // === sort(node parser.ValueTypeVector) (Vector, Annotations) === func funcSort(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { // NaN should sort to the bottom, so take descending sort with NaN first and // reverse it. - byValueSorter := vectorByReverseValueHeap(vals[0].(Vector)) + byValueSorter := vectorByReverseValueHeap(filterFloats(vals[0].(Vector))) sort.Sort(sort.Reverse(byValueSorter)) return Vector(byValueSorter), nil } @@ -486,7 +510,7 @@ func funcSort(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) func funcSortDesc(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { // NaN should sort to the bottom, so take ascending sort with NaN first and // reverse it. - byValueSorter := vectorByValueHeap(vals[0].(Vector)) + byValueSorter := vectorByValueHeap(filterFloats(vals[0].(Vector))) sort.Sort(sort.Reverse(byValueSorter)) return Vector(byValueSorter), nil } @@ -618,11 +642,27 @@ func funcRound(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper // === Scalar(node parser.ValueTypeVector) Scalar === func funcScalar(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { - v := vals[0].(Vector) - if len(v) != 1 { + var ( + v = vals[0].(Vector) + value float64 + found bool + ) + + for _, s := range v { + if s.H == nil { + if found { + // More than one float found, return NaN. + return append(enh.Out, Sample{F: math.NaN()}), nil + } + found = true + value = s.F + } + } + // Return the single float if found, otherwise return NaN. + if !found { return append(enh.Out, Sample{F: math.NaN()}), nil } - return append(enh.Out, Sample{F: v[0].F}), nil + return append(enh.Out, Sample{F: value}), nil } func aggrOverTime(vals []parser.Value, enh *EvalNodeHelper, aggrFn func(Series) float64) Vector { @@ -1612,7 +1652,7 @@ func (ev *evaluator) evalLabelReplace(ctx context.Context, args parser.Expressio if err != nil { panic(fmt.Errorf("invalid regular expression in label_replace(): %s", regexStr)) } - if !model.LabelNameRE.MatchString(dst) { + if !model.LabelName(dst).IsValid() { panic(fmt.Errorf("invalid destination label name in label_replace(): %s", dst)) } @@ -1689,6 +1729,9 @@ func (ev *evaluator) evalLabelJoin(ctx context.Context, args parser.Expressions) matrix[i].DropName = el.DropName } } + if matrix.ContainsSameLabelset() { + ev.errorf("vector cannot contain metrics with the same labelset") + } return matrix, ws } @@ -1880,16 +1923,7 @@ func (s vectorByValueHeap) Len() int { } func (s vectorByValueHeap) Less(i, j int) bool { - // We compare histograms based on their sum of observations. - // TODO(beorn7): Is that what we want? vi, vj := s[i].F, s[j].F - if s[i].H != nil { - vi = s[i].H.Sum - } - if s[j].H != nil { - vj = s[j].H.Sum - } - if math.IsNaN(vi) { return true } @@ -1919,16 +1953,7 @@ func (s vectorByReverseValueHeap) Len() int { } func (s vectorByReverseValueHeap) Less(i, j int) bool { - // We compare histograms based on their sum of observations. - // TODO(beorn7): Is that what we want? vi, vj := s[i].F, s[j].F - if s[i].H != nil { - vi = s[i].H.Sum - } - if s[j].H != nil { - vj = s[j].H.Sum - } - if math.IsNaN(vi) { return true } diff --git a/promql/promqltest/testdata/functions.test b/promql/promqltest/testdata/functions.test index 32838afe73..49ab6c50ff 100644 --- a/promql/promqltest/testdata/functions.test +++ b/promql/promqltest/testdata/functions.test @@ -466,7 +466,7 @@ eval instant at 0m label_replace(testmetric, "dst", "", "dst", ".*") eval_fail instant at 0m label_replace(testmetric, "dst", "value-$1", "src", "(.*") # label_replace fails when the destination label name is not a valid Prometheus label name. -eval_fail instant at 0m label_replace(testmetric, "invalid-label-name", "", "src", "(.*)") +eval_fail instant at 0m label_replace(testmetric, "\xff", "", "src", "(.*)") # label_replace fails when there would be duplicated identical output label sets. eval_fail instant at 0m label_replace(testmetric, "src", "", "", "") @@ -499,6 +499,8 @@ eval instant at 20s timestamp(metric) load 5m testmetric{src="a",src1="b",src2="c",dst="original-destination-value"} 0 testmetric{src="d",src1="e",src2="f",dst="original-destination-value"} 1 + dup{label="a", this="a"} 1.0 + dup{label="b", this="a"} 1.0 # label_join joins all src values in order. eval instant at 0m label_join(testmetric, "dst", "-", "src", "src1", "src2") @@ -530,6 +532,9 @@ eval instant at 0m label_join(testmetric1, "dst", ", ", "src", "src1", "src2") testmetric1{src="foo",src1="bar",src2="foobar",dst="foo, bar, foobar"} 0 testmetric1{src="fizz",src1="buzz",src2="fizzbuzz",dst="fizz, buzz, fizzbuzz"} 1 +eval_fail instant at 0m label_join(dup, "label", "", "this") + expected_fail_message vector cannot contain metrics with the same labelset + clear # Tests for vector. @@ -637,6 +642,7 @@ load 5m http_requests{job="app-server", instance="1", group="production"} 0+60x10 http_requests{job="app-server", instance="0", group="canary"} 0+70x10 http_requests{job="app-server", instance="1", group="canary"} 0+80x10 + http_requests{job="app-server", instance="2", group="canary"} {{schema:0 sum:1 count:1}}x15 eval_ordered instant at 50m sort(http_requests) http_requests{group="production", instance="0", job="api-server"} 100 @@ -1647,3 +1653,22 @@ load 1m eval range from 0 to 5m step 1m round(mixed_metric) {} _ 1 2 3 + +# Test scalar() with histograms. +load 1m + metric{type="float", l="x"} 1 + metric{type="float", l="y"} 2 + metric{type="histogram", l="x"} {{schema:0 sum:1 count:1}} + metric{type="histogram", l="x"} {{schema:0 sum:1 count:1}} + +# Two floats in the vector. +eval instant at 0m scalar(metric) + NaN + +# No floats in the vector. +eval instant at 0m scalar({type="histogram"}) + NaN + +# One float in the vector. +eval instant at 0m scalar({l="x"}) + 1 diff --git a/promql/promqltest/testdata/native_histograms.test b/promql/promqltest/testdata/native_histograms.test index f03b39a9f6..dd119c0617 100644 --- a/promql/promqltest/testdata/native_histograms.test +++ b/promql/promqltest/testdata/native_histograms.test @@ -1013,7 +1013,7 @@ eval instant at 5m sum(custom_buckets_histogram) clear -# Test 'this native histogram metric is not a gauge' warning for rate +# Test 'this native histogram metric is not a counter' warning for rate load 30s some_metric {{schema:0 sum:1 count:1 buckets:[1] counter_reset_hint:gauge}} {{schema:0 sum:2 count:2 buckets:[2] counter_reset_hint:gauge}} {{schema:0 sum:3 count:3 buckets:[3] counter_reset_hint:gauge}} @@ -1022,7 +1022,7 @@ eval_warn instant at 30s rate(some_metric[1m]) {} {{count:0.03333333333333333 sum:0.03333333333333333 buckets:[0.03333333333333333]}} # Test the case where we have more than two points for rate -eval_warn instant at 1m rate(some_metric[1m]) +eval_warn instant at 1m rate(some_metric[1m30s]) {} {{count:0.03333333333333333 sum:0.03333333333333333 buckets:[0.03333333333333333]}} clear @@ -1032,20 +1032,20 @@ load 30s some_metric {{schema:0 sum:1 count:1 buckets:[1]}} {{schema:-53 sum:1 count:1 custom_values:[5 10] buckets:[1]}} {{schema:0 sum:5 count:4 buckets:[1 2 1]}} {{schema:-53 sum:1 count:1 custom_values:[5 10] buckets:[1]}} # Start and end with exponential, with custom in the middle. -eval_warn instant at 1m rate(some_metric[1m]) +eval_warn instant at 1m rate(some_metric[1m30s]) # Should produce no results. # Start and end with custom, with exponential in the middle. -eval_warn instant at 1m30s rate(some_metric[1m]) +eval_warn instant at 1m30s rate(some_metric[1m30s]) # Should produce no results. -# Start with custom, end with exponential. -eval_warn instant at 1m rate(some_metric[1m]) - # Should produce no results. +# Start with custom, end with exponential. Return the exponential histogram divided by 30. +eval instant at 1m rate(some_metric[1m]) + {} {{schema:0 sum:0.16666666666666666 count:0.13333333333333333 buckets:[0.03333333333333333 0.06666666666666667 0.03333333333333333]}} -# Start with exponential, end with custom. -eval_warn instant at 30s rate(some_metric[1m]) - # Should produce no results. +# Start with exponential, end with custom. Return the custom buckets histogram divided by 30. +eval instant at 30s rate(some_metric[1m]) + {} {{schema:-53 sum:0.03333333333333333 count:0.03333333333333333 custom_values:[5 10] buckets:[0.03333333333333333]}} clear @@ -1179,7 +1179,10 @@ eval_info range from 0 to 6m step 6m metric2 > metric2 clear load 6m - nhcb_metric {{schema:-53 sum:1 count:1 custom_values:[2] buckets:[1]}} {{schema:-53 sum:1 count:1 custom_values:[5 10] buckets:[1]}} {{schema:-53 sum:1 count:1 custom_values:[5 10] buckets:[1]}} + nhcb_metric {{schema:-53 sum:1 count:1 custom_values:[2] buckets:[1]}} {{schema:-53 sum:1 count:1 custom_values:[2] buckets:[1]}} {{schema:-53 sum:1 count:1 custom_values:[5 10] buckets:[1]}} {{schema:-53 sum:1 count:1 custom_values:[5 10] buckets:[1]}} + +# If evaluating at 12m, the first two NHCBs have the same custom values +# while the 3rd one has different ones. eval_warn instant at 12m sum_over_time(nhcb_metric[13m]) @@ -1206,6 +1209,38 @@ eval_warn instant at 12m rate(nhcb_metric[13m]) eval instant at 12m resets(nhcb_metric[13m]) {} 1 +# Now doing the same again, but at 18m, where the first NHCB has +# different custom_values compared to the other two. This now +# works with no warning for increase() and rate(). No change +# otherwise. + +eval_warn instant at 18m sum_over_time(nhcb_metric[13m]) + +eval_warn instant at 18m avg_over_time(nhcb_metric[13m]) + +eval instant at 18m last_over_time(nhcb_metric[13m]) +nhcb_metric{} {{schema:-53 sum:1 count:1 custom_values:[5 10] buckets:[1]}} + +eval instant at 18m count_over_time(nhcb_metric[13m]) +{} 3 + +eval instant at 18m present_over_time(nhcb_metric[13m]) +{} 1 + +eval instant at 18m changes(nhcb_metric[13m]) +{} 1 + +eval_warn instant at 18m delta(nhcb_metric[13m]) + +eval instant at 18m increase(nhcb_metric[13m]) +{} {{schema:-53 count:1.0833333333333333 sum:1.0833333333333333 custom_values:[5 10] buckets:[1.0833333333333333]}} + +eval instant at 18m rate(nhcb_metric[13m]) +{} {{schema:-53 count:0.0013888888888888887 sum:0.0013888888888888887 custom_values:[5 10] buckets:[0.0013888888888888887]}} + +eval instant at 18m resets(nhcb_metric[13m]) +{} 1 + clear load 1m diff --git a/scrape/scrape_test.go b/scrape/scrape_test.go index 26117ffd08..2185d028fb 100644 --- a/scrape/scrape_test.go +++ b/scrape/scrape_test.go @@ -25,6 +25,8 @@ import ( "net/http" "net/http/httptest" "net/url" + "os" + "sort" "strconv" "strings" "sync" @@ -1386,8 +1388,17 @@ func TestScrapeLoopFailLegacyUnderUTF8(t *testing.T) { require.Equal(t, 1, seriesAdded) } -func makeTestMetrics(n int) []byte { - // Construct a metrics string to parse +func readTextParseTestMetrics(t testing.TB) []byte { + t.Helper() + + b, err := os.ReadFile("../model/textparse/testdata/alltypes.237mfs.prom.txt") + if err != nil { + t.Fatal(err) + } + return b +} + +func makeTestGauges(n int) []byte { sb := bytes.Buffer{} fmt.Fprintf(&sb, "# TYPE metric_a gauge\n") fmt.Fprintf(&sb, "# HELP metric_a help text\n") @@ -1401,59 +1412,111 @@ func makeTestMetrics(n int) []byte { func promTextToProto(tb testing.TB, text []byte) []byte { tb.Helper() - d := expfmt.NewDecoder(bytes.NewReader(text), expfmt.TextVersion) - - pb := &dto.MetricFamily{} - if err := d.Decode(pb); err != nil { - tb.Fatal(err) - } - o, err := proto.Marshal(pb) + var p expfmt.TextParser + fams, err := p.TextToMetricFamilies(bytes.NewReader(text)) if err != nil { tb.Fatal(err) } + // Order by name for the deterministic tests. + var names []string + for n := range fams { + names = append(names, n) + } + sort.Strings(names) + buf := bytes.Buffer{} - // Write first length, then binary protobuf. - varintBuf := binary.AppendUvarint(nil, uint64(len(o))) - buf.Write(varintBuf) - buf.Write(o) + for _, n := range names { + o, err := proto.Marshal(fams[n]) + if err != nil { + tb.Fatal(err) + } + + // Write first length, then binary protobuf. + varintBuf := binary.AppendUvarint(nil, uint64(len(o))) + buf.Write(varintBuf) + buf.Write(o) + } return buf.Bytes() } +func TestPromTextToProto(t *testing.T) { + metricsText := readTextParseTestMetrics(t) + // TODO(bwplotka): Windows adds \r for new lines which is + // not handled correctly in the expfmt parser, fix it. + metricsText = bytes.ReplaceAll(metricsText, []byte("\r"), nil) + + metricsProto := promTextToProto(t, metricsText) + d := expfmt.NewDecoder(bytes.NewReader(metricsProto), expfmt.NewFormat(expfmt.TypeProtoDelim)) + + var got []string + for { + mf := &dto.MetricFamily{} + if err := d.Decode(mf); err != nil { + if errors.Is(err, io.EOF) { + break + } + t.Fatal(err) + } + got = append(got, mf.GetName()) + } + require.Len(t, got, 237) + // Check a few to see if those are not dups. + require.Equal(t, "go_gc_cycles_automatic_gc_cycles_total", got[0]) + require.Equal(t, "prometheus_sd_kuma_fetch_duration_seconds", got[128]) + require.Equal(t, "promhttp_metric_handler_requests_total", got[236]) +} + +// BenchmarkScrapeLoopAppend benchmarks a core append function in a scrapeLoop +// that creates a new parser and goes through a byte slice from a single scrape. +// Benchmark compares append function run across 2 dimensions: +// *`data`: different sizes of metrics scraped e.g. one big gauge metric family +// with a thousand series and more realistic scenario with common types. +// *`fmt`: different scrape formats which will benchmark different parsers e.g. +// promtext, omtext and promproto. +// +// Recommended CLI invocation: /* - export bench=scrape-loop-v1 && go test \ + export bench=append-v1 && go test ./scrape/... \ -run '^$' -bench '^BenchmarkScrapeLoopAppend' \ -benchtime 5s -count 6 -cpu 2 -timeout 999m \ | tee ${bench}.txt */ func BenchmarkScrapeLoopAppend(b *testing.B) { - metricsText := makeTestMetrics(100) - - // Create proto representation. - metricsProto := promTextToProto(b, metricsText) - - for _, bcase := range []struct { - name string - contentType string - parsable []byte + for _, data := range []struct { + name string + parsableText []byte }{ - {name: "PromText", contentType: "text/plain", parsable: metricsText}, - {name: "OMText", contentType: "application/openmetrics-text", parsable: metricsText}, - {name: "PromProto", contentType: "application/vnd.google.protobuf", parsable: metricsProto}, + {name: "1Fam1000Gauges", parsableText: makeTestGauges(2000)}, // ~68.1 KB, ~77.9 KB in proto. + {name: "237FamsAllTypes", parsableText: readTextParseTestMetrics(b)}, // ~185.7 KB, ~70.6 KB in proto. } { - b.Run(fmt.Sprintf("fmt=%v", bcase.name), func(b *testing.B) { - ctx, sl := simpleTestScrapeLoop(b) + b.Run(fmt.Sprintf("data=%v", data.name), func(b *testing.B) { + metricsProto := promTextToProto(b, data.parsableText) - slApp := sl.appender(ctx) - ts := time.Time{} + for _, bcase := range []struct { + name string + contentType string + parsable []byte + }{ + {name: "PromText", contentType: "text/plain", parsable: data.parsableText}, + {name: "OMText", contentType: "application/openmetrics-text", parsable: data.parsableText}, + {name: "PromProto", contentType: "application/vnd.google.protobuf", parsable: metricsProto}, + } { + b.Run(fmt.Sprintf("fmt=%v", bcase.name), func(b *testing.B) { + ctx, sl := simpleTestScrapeLoop(b) - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - ts = ts.Add(time.Second) - _, _, _, err := sl.append(slApp, bcase.parsable, bcase.contentType, ts) - if err != nil { - b.Fatal(err) - } + slApp := sl.appender(ctx) + ts := time.Time{} + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + ts = ts.Add(time.Second) + _, _, _, err := sl.append(slApp, bcase.parsable, bcase.contentType, ts) + if err != nil { + b.Fatal(err) + } + } + }) } }) } @@ -4474,7 +4537,7 @@ func TestScrapeLoopCompression(t *testing.T) { simpleStorage := teststorage.New(t) defer simpleStorage.Close() - metricsText := makeTestMetrics(10) + metricsText := makeTestGauges(10) for _, tc := range []struct { enableCompression bool diff --git a/scripts/golangci-lint.yml b/scripts/golangci-lint.yml index 0c00c410a4..def9007ac6 100644 --- a/scripts/golangci-lint.yml +++ b/scripts/golangci-lint.yml @@ -26,14 +26,14 @@ jobs: - name: Checkout repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Go - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0 + uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0 with: go-version: 1.23.x - name: Install snmp_exporter/generator dependencies run: sudo apt-get update && sudo apt-get -y install libsnmp-dev if: github.repository == 'prometheus/snmp_exporter' - name: Lint - uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # v6.1.1 + uses: golangci/golangci-lint-action@ec5d18412c0aeab7936cb16880d708ba2a64e1ae # v6.2.0 with: args: --verbose version: v1.63.4 diff --git a/tsdb/db.go b/tsdb/db.go index 2d35e3fb00..9ab150c5b4 100644 --- a/tsdb/db.go +++ b/tsdb/db.go @@ -992,9 +992,14 @@ func open(dir string, l *slog.Logger, r prometheus.Registerer, opts *Options, rn db.metrics.maxBytes.Set(float64(maxBytes)) db.metrics.retentionDuration.Set((time.Duration(opts.RetentionDuration) * time.Millisecond).Seconds()) + // Calling db.reload() calls db.reloadBlocks() which requires cmtx to be locked. + db.cmtx.Lock() if err := db.reload(); err != nil { + db.cmtx.Unlock() return nil, err } + db.cmtx.Unlock() + // Set the min valid time for the ingested samples // to be no lower than the maxt of the last block. minValidTime := int64(math.MinInt64) @@ -1363,6 +1368,7 @@ func (db *DB) CompactOOOHead(ctx context.Context) error { // Callback for testing. var compactOOOHeadTestingCallback func() +// The db.cmtx mutex should be held before calling this method. func (db *DB) compactOOOHead(ctx context.Context) error { if !db.oooWasEnabled.Load() { return nil @@ -1417,6 +1423,7 @@ func (db *DB) compactOOOHead(ctx context.Context) error { // compactOOO creates a new block per possible block range in the compactor's directory from the OOO Head given. // Each ULID in the result corresponds to a block in a unique time range. +// The db.cmtx mutex should be held before calling this method. func (db *DB) compactOOO(dest string, oooHead *OOOCompactionHead) (_ []ulid.ULID, err error) { start := time.Now() @@ -1461,7 +1468,7 @@ func (db *DB) compactOOO(dest string, oooHead *OOOCompactionHead) (_ []ulid.ULID } // compactHead compacts the given RangeHead. -// The compaction mutex should be held before calling this method. +// The db.cmtx should be held before calling this method. func (db *DB) compactHead(head *RangeHead) error { uids, err := db.compactor.Write(db.dir, head, head.MinTime(), head.BlockMaxTime(), nil) if err != nil { @@ -1487,7 +1494,7 @@ func (db *DB) compactHead(head *RangeHead) error { } // compactBlocks compacts all the eligible on-disk blocks. -// The compaction mutex should be held before calling this method. +// The db.cmtx should be held before calling this method. func (db *DB) compactBlocks() (err error) { // Check for compactions of multiple blocks. for { @@ -1544,6 +1551,7 @@ func getBlock(allBlocks []*Block, id ulid.ULID) (*Block, bool) { } // reload reloads blocks and truncates the head and its WAL. +// The db.cmtx mutex should be held before calling this method. func (db *DB) reload() error { if err := db.reloadBlocks(); err != nil { return fmt.Errorf("reloadBlocks: %w", err) @@ -1560,6 +1568,7 @@ func (db *DB) reload() error { // reloadBlocks reloads blocks without touching head. // Blocks that are obsolete due to replacement or retention will be deleted. +// The db.cmtx mutex should be held before calling this method. func (db *DB) reloadBlocks() (err error) { defer func() { if err != nil { @@ -1568,13 +1577,9 @@ func (db *DB) reloadBlocks() (err error) { db.metrics.reloads.Inc() }() - // Now that we reload TSDB every minute, there is a high chance for a race condition with a reload - // triggered by CleanTombstones(). We need to lock the reload to avoid the situation where - // a normal reload and CleanTombstones try to delete the same block. - db.mtx.Lock() - defer db.mtx.Unlock() - + db.mtx.RLock() loadable, corrupted, err := openBlocks(db.logger, db.dir, db.blocks, db.chunkPool, db.opts.PostingsDecoderFactory) + db.mtx.RUnlock() if err != nil { return err } @@ -1600,11 +1605,13 @@ func (db *DB) reloadBlocks() (err error) { if len(corrupted) > 0 { // Corrupted but no child loaded for it. // Close all new blocks to release the lock for windows. + db.mtx.RLock() for _, block := range loadable { if _, open := getBlock(db.blocks, block.Meta().ULID); !open { block.Close() } } + db.mtx.RUnlock() errs := tsdb_errors.NewMulti() for ulid, err := range corrupted { if err != nil { @@ -1643,8 +1650,10 @@ func (db *DB) reloadBlocks() (err error) { }) // Swap new blocks first for subsequently created readers to be seen. + db.mtx.Lock() oldBlocks := db.blocks db.blocks = toLoad + db.mtx.Unlock() // Only check overlapping blocks when overlapping compaction is enabled. if db.opts.EnableOverlappingCompaction { diff --git a/tsdb/db_test.go b/tsdb/db_test.go index 7dc60a7304..826be61a42 100644 --- a/tsdb/db_test.go +++ b/tsdb/db_test.go @@ -1352,61 +1352,6 @@ func TestTombstoneCleanFail(t *testing.T) { require.Len(t, intersection(oldBlockDirs, actualBlockDirs), len(actualBlockDirs)-1) } -// TestTombstoneCleanRetentionLimitsRace tests that a CleanTombstones operation -// and retention limit policies, when triggered at the same time, -// won't race against each other. -func TestTombstoneCleanRetentionLimitsRace(t *testing.T) { - if testing.Short() { - t.Skip("skipping test in short mode.") - } - - opts := DefaultOptions() - var wg sync.WaitGroup - - // We want to make sure that a race doesn't happen when a normal reload and a CleanTombstones() - // reload try to delete the same block. Without the correct lock placement, it can happen if a - // block is marked for deletion due to retention limits and also has tombstones to be cleaned at - // the same time. - // - // That is something tricky to trigger, so let's try several times just to make sure. - for i := 0; i < 20; i++ { - t.Run(fmt.Sprintf("iteration%d", i), func(t *testing.T) { - db := openTestDB(t, opts, nil) - totalBlocks := 20 - dbDir := db.Dir() - // Generate some blocks with old mint (near epoch). - for j := 0; j < totalBlocks; j++ { - blockDir := createBlock(t, dbDir, genSeries(10, 1, int64(j), int64(j)+1)) - block, err := OpenBlock(nil, blockDir, nil, nil) - require.NoError(t, err) - // Cover block with tombstones so it can be deleted with CleanTombstones() as well. - tomb := tombstones.NewMemTombstones() - tomb.AddInterval(0, tombstones.Interval{Mint: int64(j), Maxt: int64(j) + 1}) - block.tombstones = tomb - - db.blocks = append(db.blocks, block) - } - - wg.Add(2) - // Run reload and CleanTombstones together, with a small time window randomization - go func() { - defer wg.Done() - time.Sleep(time.Duration(rand.Float64() * 100 * float64(time.Millisecond))) - require.NoError(t, db.reloadBlocks()) - }() - go func() { - defer wg.Done() - time.Sleep(time.Duration(rand.Float64() * 100 * float64(time.Millisecond))) - require.NoError(t, db.CleanTombstones()) - }() - - wg.Wait() - - require.NoError(t, db.Close()) - }) - } -} - func intersection(oldBlocks, actualBlocks []string) (intersection []string) { hash := make(map[string]bool) for _, e := range oldBlocks { diff --git a/tsdb/head_read.go b/tsdb/head_read.go index b95257c28a..438c7e6a4d 100644 --- a/tsdb/head_read.go +++ b/tsdb/head_read.go @@ -117,15 +117,19 @@ func (h *headIndexReader) PostingsForAllLabelValues(ctx context.Context, name st func (h *headIndexReader) SortedPostings(p index.Postings) index.Postings { series := make([]*memSeries, 0, 128) + notFoundSeriesCount := 0 // Fetch all the series only once. for p.Next() { s := h.head.series.getByID(chunks.HeadSeriesRef(p.At())) if s == nil { - h.head.logger.Debug("Looked up series not found") + notFoundSeriesCount++ } else { series = append(series, s) } } + if notFoundSeriesCount > 0 { + h.head.logger.Debug("Looked up series not found", "count", notFoundSeriesCount) + } if err := p.Err(); err != nil { return index.ErrPostings(fmt.Errorf("expand postings: %w", err)) } @@ -150,11 +154,12 @@ func (h *headIndexReader) ShardedPostings(p index.Postings, shardIndex, shardCou } out := make([]storage.SeriesRef, 0, 128) + notFoundSeriesCount := 0 for p.Next() { s := h.head.series.getByID(chunks.HeadSeriesRef(p.At())) if s == nil { - h.head.logger.Debug("Looked up series not found") + notFoundSeriesCount++ continue } @@ -165,6 +170,9 @@ func (h *headIndexReader) ShardedPostings(p index.Postings, shardIndex, shardCou out = append(out, storage.SeriesRef(s.ref)) } + if notFoundSeriesCount > 0 { + h.head.logger.Debug("Looked up series not found", "count", notFoundSeriesCount) + } return index.NewListPostings(out) } diff --git a/util/httputil/compression.go b/util/httputil/compression.go index 9a8a666453..d5bedb7fa9 100644 --- a/util/httputil/compression.go +++ b/util/httputil/compression.go @@ -56,8 +56,13 @@ func (c *compressedResponseWriter) Close() { // Constructs a new compressedResponseWriter based on client request headers. func newCompressedResponseWriter(writer http.ResponseWriter, req *http.Request) *compressedResponseWriter { - encodings := strings.Split(req.Header.Get(acceptEncodingHeader), ",") - for _, encoding := range encodings { + raw := req.Header.Get(acceptEncodingHeader) + var ( + encoding string + commaFound bool + ) + for { + encoding, raw, commaFound = strings.Cut(raw, ",") switch strings.TrimSpace(encoding) { case gzipEncoding: writer.Header().Set(contentEncodingHeader, gzipEncoding) @@ -72,6 +77,9 @@ func newCompressedResponseWriter(writer http.ResponseWriter, req *http.Request) writer: zlib.NewWriter(writer), } } + if !commaFound { + break + } } return &compressedResponseWriter{ ResponseWriter: writer, diff --git a/util/httputil/compression_test.go b/util/httputil/compression_test.go index e166c7de79..fd3f1f66d4 100644 --- a/util/httputil/compression_test.go +++ b/util/httputil/compression_test.go @@ -18,6 +18,7 @@ import ( "io" "net/http" "net/http/httptest" + "strings" "testing" "github.com/klauspost/compress/gzip" @@ -72,6 +73,17 @@ func TestCompressionHandler_PlainText(t *testing.T) { require.Equal(t, expected, actual, "expected response with content") } +func BenchmarkNewCompressionHandler_MaliciousAcceptEncoding(b *testing.B) { + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/whatever", nil) + req.Header.Set("Accept-Encoding", strings.Repeat(",", http.DefaultMaxHeaderBytes)) + b.ReportAllocs() + b.ResetTimer() + for range b.N { + newCompressedResponseWriter(rec, req) + } +} + func TestCompressionHandler_Gzip(t *testing.T) { tearDown := setup() defer tearDown() diff --git a/web/ui/mantine-ui/src/pages/query/uPlotChartHelpers.ts b/web/ui/mantine-ui/src/pages/query/uPlotChartHelpers.ts index e9dad3665f..ac77fee596 100644 --- a/web/ui/mantine-ui/src/pages/query/uPlotChartHelpers.ts +++ b/web/ui/mantine-ui/src/pages/query/uPlotChartHelpers.ts @@ -332,6 +332,7 @@ export const getUPlotOptions = ( axes: [ // X axis (time). { + space: 80, labelSize: 20, stroke: light ? "#333" : "#eee", ticks: { @@ -343,6 +344,33 @@ export const getUPlotOptions = ( width: 2, dash: [], }, + values: [ + // See https://github.com/leeoniya/uPlot/tree/master/docs#axis--grid-opts and https://github.com/leeoniya/uPlot/issues/83. + // + // We want to achieve 24h-based time formatting instead of the default AM/PM-based time formatting. + // We also want to render dates in an unambiguous format that uses the abbreviated month name instead of a US-centric DD/MM/YYYY format. + // + // The "tick incr" column defines the breakpoint in seconds at which the format changes. + // The "default" column defines the default format for a tick at this breakpoint. + // The "year"/"month"/"day"/"hour"/"min"/"sec" columns define additional values to display for year/month/day/... rollovers occurring around a tick. + // The "mode" column value "1" means that rollover values will be concatenated with the default format (instead of replacing it). + // + // tick incr default year month day hour min sec mode + // prettier-ignore + [3600 * 24 * 365, "{YYYY}", null, null, null, null, null, null, 1], + // prettier-ignore + [3600 * 24 * 28, "{MMM}", "\n{YYYY}", null, null, null, null, null, 1], + // prettier-ignore + [3600 * 24, "{MMM} {D}", "\n{YYYY}", null, null, null, null, null, 1], + // prettier-ignore + [3600, "{HH}:{mm}", "\n{MMM} {D} '{YY}", null, "\n{MMM} {D}", null, null, null, 1], + // prettier-ignore + [60, "{HH}:{mm}", "\n{MMM} {D} '{YY}", null, "\n{MMM} {D}", null, null, null, 1], + // prettier-ignore + [1, "{HH}:{mm}:{ss}", "\n{MMM} {D} '{YY}", null, "\n{MMM} {D}", null, null, null, 1], + // prettier-ignore + [0.001, "{HH}:{mm}:{ss}.{fff}", "\n{MMM} {D} '{YY}", null, "\n{MMM} {D}", null, null, null, 1], + ], }, // Y axis (sample value). { @@ -382,7 +410,10 @@ export const getUPlotOptions = ( (self: uPlot) => { // Disallow sub-second zoom as this cause inconsistenices in the X axis in uPlot. const leftVal = self.posToVal(self.select.left, "x"); - const rightVal = Math.max(self.posToVal(self.select.left + self.select.width, "x"), leftVal + 1); + const rightVal = Math.max( + self.posToVal(self.select.left + self.select.width, "x"), + leftVal + 1 + ); onSelectRange(leftVal, rightVal); },