From 8b1bd7d6c302025d08fa4ead3a19a3425a031ea2 Mon Sep 17 00:00:00 2001 From: Julius Volz Date: Wed, 15 Oct 2025 15:08:07 +0200 Subject: [PATCH] ui: Allow viewing detailed relabeling steps for each discovered target This adds: * A `ScrapePoolConfig()` method to the scrape manager that allows getting the scrape config for a given pool. * An API endpoint at `/api/v1/targets/relabel_steps` that takes a pool name and a label set of a target and returns a detailed list of applied relabeling rules and their output for each step. * A "show relabeling" link/button for each target on the discovery page that shows the detailed flow of all relabeling rules (based on the API response) for that target. Note that this changes the JSON encoding of the relabeling rule config struct to output the original snake_case (instead of camelCase) field names, and before merging, we need to be sure that's ok :) See my comment about that at https://github.com/prometheus/prometheus/pull/15383#issuecomment-3405591487 Fixes https://github.com/prometheus/prometheus/issues/17283 Signed-off-by: Julius Volz --- docs/querying/api.md | 58 ++++++ model/relabel/relabel.go | 4 +- scrape/manager.go | 12 ++ web/api/v1/api.go | 46 +++++ web/api/v1/api_test.go | 58 +++++- web/api/v1/errors_test.go | 4 + .../src/api/responseTypes/relabel_steps.ts | 13 ++ .../pages/service-discovery/RelabelSteps.tsx | 188 ++++++++++++++++++ .../ServiceDiscoveryPoolsList.tsx | 70 ++++++- 9 files changed, 439 insertions(+), 14 deletions(-) create mode 100644 web/ui/mantine-ui/src/api/responseTypes/relabel_steps.ts create mode 100644 web/ui/mantine-ui/src/pages/service-discovery/RelabelSteps.tsx diff --git a/docs/querying/api.md b/docs/querying/api.md index 1ff3beef17..65c5ed476d 100644 --- a/docs/querying/api.md +++ b/docs/querying/api.md @@ -810,6 +810,64 @@ curl 'http://localhost:9090/api/v1/targets?scrapePool=node_exporter' } ``` +## Relabel steps + +This endpoint is **experimental** and might change in the future. It is currently only meant to be used by Prometheus' own web UI, and the endpoint name and exact format returned may change from one Prometheus version to another. It may also be removed again in case it is no longer needed by the UI. + +The following endpoint returns a step-by-step list of relabeling rules and their effects on a given target's label set. + +``` +GET /api/v1/targets/relabel_steps +``` + +URL query parameters: +- `scrapePool=`: The scrape pool name of the target, used to determine the relabeling rules to apply. Required. +- `labels=`: A JSON object containing the label set of the target before any relabeling is applied. Required. + +The following example returns the relabeling steps for a discovered target in the `prometheus` scrape pool with the label set `{"__address__": "localhost:9090", "job": "prometheus"}`: + +```bash +curl -g 'http://localhost:9090/api/v1/targets/relabel_steps?scrapePool=prometheus&labels={"__address__":"localhost:9090","job":"prometheus"}' +``` + +```json +{ + "data" : { + "steps" : [ + { + "keep" : true, + "output" : { + "__address__" : "localhost:9090", + "env" : "development", + "job" : "prometheus" + }, + "rule" : { + "action" : "replace", + "regex" : "(.*)", + "replacement" : "development", + "separator" : ";", + "target_label" : "env" + } + }, + { + "keep" : false, + "output" : {}, + "rule" : { + "action" : "drop", + "regex" : "localhost:.*", + "replacement" : "$1", + "separator" : ";", + "source_labels" : [ + "__address__" + ] + } + } + ] + }, + "status" : "success" +} +``` + ## Rules The `/rules` API endpoint returns a list of alerting and recording rules that diff --git a/model/relabel/relabel.go b/model/relabel/relabel.go index c7c5439d54..95cfeecc02 100644 --- a/model/relabel/relabel.go +++ b/model/relabel/relabel.go @@ -86,7 +86,7 @@ func (a *Action) UnmarshalYAML(unmarshal func(any) error) error { type Config struct { // A list of labels from which values are taken and concatenated // with the configured separator in order. - SourceLabels model.LabelNames `yaml:"source_labels,flow,omitempty" json:"sourceLabels,omitempty"` + SourceLabels model.LabelNames `yaml:"source_labels,flow,omitempty" json:"source_labels,omitempty"` // Separator is the string between concatenated values from the source labels. Separator string `yaml:"separator,omitempty" json:"separator,omitempty"` // Regex against which the concatenation is matched. @@ -95,7 +95,7 @@ type Config struct { Modulus uint64 `yaml:"modulus,omitempty" json:"modulus,omitempty"` // TargetLabel is the label to which the resulting string is written in a replacement. // Regexp interpolation is allowed for the replace action. - TargetLabel string `yaml:"target_label,omitempty" json:"targetLabel,omitempty"` + TargetLabel string `yaml:"target_label,omitempty" json:"target_label,omitempty"` // Replacement is the regex replacement pattern to be used. Replacement string `yaml:"replacement,omitempty" json:"replacement,omitempty"` // Action is the action to be performed for the relabeling. diff --git a/scrape/manager.go b/scrape/manager.go index 3375031212..fffdf9c4ec 100644 --- a/scrape/manager.go +++ b/scrape/manager.go @@ -399,3 +399,15 @@ func (m *Manager) TargetsDroppedCounts() map[string]int { } return counts } + +func (m *Manager) ScrapePoolConfig(scrapePool string) (*config.ScrapeConfig, error) { + m.mtxScrape.Lock() + defer m.mtxScrape.Unlock() + + sp, ok := m.scrapePools[scrapePool] + if !ok { + return nil, fmt.Errorf("scrape pool %q not found", scrapePool) + } + + return sp.config, nil +} diff --git a/web/api/v1/api.go b/web/api/v1/api.go index 7ea81e70c6..c6e37b7c13 100644 --- a/web/api/v1/api.go +++ b/web/api/v1/api.go @@ -44,6 +44,7 @@ import ( "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/metadata" + "github.com/prometheus/prometheus/model/relabel" "github.com/prometheus/prometheus/model/timestamp" "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/promql/parser" @@ -129,6 +130,7 @@ type TargetRetriever interface { TargetsActive() map[string][]*scrape.Target TargetsDropped() map[string][]*scrape.Target TargetsDroppedCounts() map[string]int + ScrapePoolConfig(string) (*config.ScrapeConfig, error) } // AlertmanagerRetriever provides a list of all/dropped AlertManager URLs. @@ -429,6 +431,7 @@ func (api *API) Register(r *route.Router) { r.Get("/scrape_pools", wrap(api.scrapePools)) r.Get("/targets", wrap(api.targets)) r.Get("/targets/metadata", wrap(api.targetMetadata)) + r.Get("/targets/relabel_steps", wrap(api.targetRelabelSteps)) r.Get("/alertmanagers", wrapAgent(api.alertmanagers)) r.Get("/metadata", wrap(api.metricMetadata)) @@ -1303,6 +1306,49 @@ type metricMetadata struct { Unit string `json:"unit"` } +type RelabelStep struct { + Rule *relabel.Config `json:"rule"` + Output labels.Labels `json:"output"` + Keep bool `json:"keep"` +} + +type RelabelStepsResponse struct { + Steps []RelabelStep `json:"steps"` +} + +func (api *API) targetRelabelSteps(r *http.Request) apiFuncResult { + scrapePool := r.FormValue("scrapePool") + if scrapePool == "" { + return apiFuncResult{nil, &apiError{errorBadData, errors.New("no scrapePool parameter provided")}, nil, nil} + } + labelsJSON := r.FormValue("labels") + if labelsJSON == "" { + return apiFuncResult{nil, &apiError{errorBadData, errors.New("no labels parameter provided")}, nil, nil} + } + var lbls labels.Labels + if err := json.Unmarshal([]byte(labelsJSON), &lbls); err != nil { + return apiFuncResult{nil, &apiError{errorBadData, fmt.Errorf("error parsing labels: %w", err)}, nil, nil} + } + + scrapeConfig, err := api.targetRetriever(r.Context()).ScrapePoolConfig(scrapePool) + if err != nil { + return apiFuncResult{nil, &apiError{errorBadData, fmt.Errorf("error retrieving scrape config: %w", err)}, nil, nil} + } + + rules := scrapeConfig.RelabelConfigs + steps := make([]RelabelStep, len(rules)) + for i, rule := range rules { + outLabels, keep := relabel.Process(lbls, rules[:i+1]...) + steps[i] = RelabelStep{ + Rule: rule, + Output: outLabels, + Keep: keep, + } + } + + return apiFuncResult{&RelabelStepsResponse{Steps: steps}, nil, nil, nil} +} + // AlertmanagerDiscovery has all the active Alertmanagers. type AlertmanagerDiscovery struct { ActiveAlertmanagers []*AlertmanagerTarget `json:"activeAlertmanagers"` diff --git a/web/api/v1/api_test.go b/web/api/v1/api_test.go index 5c7bfbda05..af06ee1b41 100644 --- a/web/api/v1/api_test.go +++ b/web/api/v1/api_test.go @@ -31,6 +31,9 @@ import ( "testing" "time" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/grafana/regexp" jsoniter "github.com/json-iterator/go" "github.com/oklog/ulid/v2" "github.com/prometheus/client_golang/prometheus" @@ -44,6 +47,7 @@ import ( "github.com/prometheus/prometheus/model/exemplar" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/metadata" + "github.com/prometheus/prometheus/model/relabel" "github.com/prometheus/prometheus/model/timestamp" "github.com/prometheus/prometheus/prompb" "github.com/prometheus/prometheus/promql" @@ -162,6 +166,25 @@ func (t testTargetRetriever) TargetsDroppedCounts() map[string]int { return r } +func (testTargetRetriever) ScrapePoolConfig(_ string) (*config.ScrapeConfig, error) { + return &config.ScrapeConfig{ + RelabelConfigs: []*relabel.Config{ + { + Action: relabel.Replace, + Replacement: "example.com:443", + TargetLabel: "__address__", + Regex: relabel.MustNewRegexp(""), + NameValidationScheme: model.LegacyValidation, + }, + { + Action: relabel.Drop, + SourceLabels: []model.LabelName{"__address__"}, + Regex: relabel.MustNewRegexp(`example\.com:.*`), + }, + }, + }, nil +} + func (t *testTargetRetriever) SetMetadataStoreForTargets(identifier string, metadata scrape.MetricMetadataStore) error { targets, ok := t.activeTargets[identifier] @@ -1883,6 +1906,37 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E DroppedTargetCounts: map[string]int{"blackbox": 1}, }, }, + { + endpoint: api.targetRelabelSteps, + query: url.Values{"scrapePool": []string{"testpool"}, "labels": []string{`{"job":"test","__address__":"localhost:9090"}`}}, + response: &RelabelStepsResponse{ + Steps: []RelabelStep{ + { + Rule: &relabel.Config{ + Action: relabel.Replace, + Replacement: "example.com:443", + TargetLabel: "__address__", + Regex: relabel.MustNewRegexp(""), + NameValidationScheme: model.LegacyValidation, + }, + Output: labels.FromMap(map[string]string{ + "job": "test", + "__address__": "example.com:443", + }), + Keep: true, + }, + { + Rule: &relabel.Config{ + Action: relabel.Drop, + SourceLabels: []model.LabelName{"__address__"}, + Regex: relabel.MustNewRegexp(`example\.com:.*`), + }, + Output: labels.EmptyLabels(), + Keep: false, + }, + }, + }, + }, // With a matching metric. { endpoint: api.targetMetadata, @@ -3772,7 +3826,9 @@ func assertAPIError(t *testing.T, got *apiError, exp errorType) { func assertAPIResponse(t *testing.T, got, exp any) { t.Helper() - testutil.RequireEqual(t, exp, got) + testutil.RequireEqualWithOptions(t, exp, got, []cmp.Option{ + cmpopts.IgnoreUnexported(regexp.Regexp{}), + }) } func assertAPIResponseLength(t *testing.T, got any, expLen int) { diff --git a/web/api/v1/errors_test.go b/web/api/v1/errors_test.go index fd2e92a850..551a3ad19c 100644 --- a/web/api/v1/errors_test.go +++ b/web/api/v1/errors_test.go @@ -264,6 +264,10 @@ func (DummyTargetRetriever) TargetsDroppedCounts() map[string]int { return nil } +func (DummyTargetRetriever) ScrapePoolConfig(_ string) (*config.ScrapeConfig, error) { + return nil, errors.New("not implemented") +} + // DummyAlertmanagerRetriever implements AlertmanagerRetriever. type DummyAlertmanagerRetriever struct{} diff --git a/web/ui/mantine-ui/src/api/responseTypes/relabel_steps.ts b/web/ui/mantine-ui/src/api/responseTypes/relabel_steps.ts new file mode 100644 index 0000000000..2240a9c91d --- /dev/null +++ b/web/ui/mantine-ui/src/api/responseTypes/relabel_steps.ts @@ -0,0 +1,13 @@ +import { Labels } from "./targets"; + +export type RelabelStep = { + rule: { [key: string]: unknown }; + output: Labels; + keep: boolean; +}; + +// Result type for /api/v1/relabel_steps endpoint. +// See: https://prometheus.io/docs/prometheus/latest/querying/api/#relabel_steps +export type RelabelStepsResult = { + steps: RelabelStep[]; +}; diff --git a/web/ui/mantine-ui/src/pages/service-discovery/RelabelSteps.tsx b/web/ui/mantine-ui/src/pages/service-discovery/RelabelSteps.tsx new file mode 100644 index 0000000000..927ce43be4 --- /dev/null +++ b/web/ui/mantine-ui/src/pages/service-discovery/RelabelSteps.tsx @@ -0,0 +1,188 @@ +import { em, Group, Stack, Table, Text } from "@mantine/core"; +import { useSuspenseAPIQuery } from "../../api/api"; +import { RelabelStepsResult } from "../../api/responseTypes/relabel_steps"; +import { Labels } from "../../api/responseTypes/targets"; +import React from "react"; +import { + IconArrowDown, + IconCircleMinus, + IconCirclePlus, + IconReplace, + IconTags, +} from "@tabler/icons-react"; + +const iconStyle = { width: em(18), height: em(18), verticalAlign: "middle" }; + +const ruleTable = (rule: { [key: string]: unknown }, idx: number) => { + return ( + + + + + + Rule {idx + 1} + + + + + + {Object.entries(rule) + .sort(([a], [b]) => { + // Sort by a predefined order for known fields, otherwise alphabetically. + const sortedRuleFieldNames: string[] = [ + "action", + "source_labels", + "regex", + "modulus", + "replacement", + "target_label", + ]; + const ai = sortedRuleFieldNames.indexOf(a); + const bi = sortedRuleFieldNames.indexOf(b); + if (ai === -1 && bi === -1) { + return a.localeCompare(b); + } + if (ai === -1) { + return 1; + } + if (bi === -1) { + return -1; + } + return ai - bi; + }) + .map(([k, v]) => ( + + + {k} + + + + {typeof v === "object" ? JSON.stringify(v) : String(v)} + + + + ))} + +
+ ); +}; + +const labelsTable = (labels: Labels, prevLabels: Labels | null) => { + if (labels === null) { + return dropped; + } + + return ( + + + + + + Labels + + + + + + {Object.entries(labels) + .concat( + prevLabels + ? Object.entries(prevLabels).filter( + ([k]) => labels[k] === undefined + ) + : [] + ) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([k, v]) => { + const added = prevLabels !== null && prevLabels[k] === undefined; + const changed = + prevLabels !== null && !added && prevLabels[k] !== v; + const removed = + prevLabels !== null && + !changed && + prevLabels[k] !== undefined && + labels[k] === undefined; + return ( + + + {added ? ( + + ) : changed ? ( + + ) : removed ? ( + + ) : null} + + + {k} + + + {v} + + + ); + })} + +
+ ); +}; + +const RelabelSteps: React.FC<{ + labels: Labels; + pool: string; +}> = ({ labels, pool }) => { + const { data } = useSuspenseAPIQuery({ + path: `/targets/relabel_steps`, + params: { + labels: JSON.stringify(labels), + scrapePool: pool, + }, + }); + + return ( + + {labelsTable(labels, null)} + {data.data.steps.map((step, idx) => ( + + + {ruleTable(step.rule, idx)} + + {step.keep ? ( + labelsTable( + step.output, + idx === 0 ? labels : data.data.steps[idx - 1].output + ) + ) : ( + dropped + )} + + ))} + + ); +}; + +export default RelabelSteps; diff --git a/web/ui/mantine-ui/src/pages/service-discovery/ServiceDiscoveryPoolsList.tsx b/web/ui/mantine-ui/src/pages/service-discovery/ServiceDiscoveryPoolsList.tsx index 8bfbebf698..4ba522d9da 100644 --- a/web/ui/mantine-ui/src/pages/service-discovery/ServiceDiscoveryPoolsList.tsx +++ b/web/ui/mantine-ui/src/pages/service-discovery/ServiceDiscoveryPoolsList.tsx @@ -2,8 +2,11 @@ import { Accordion, Alert, Anchor, + Box, Group, + Modal, RingProgress, + Skeleton, Stack, Table, Text, @@ -17,7 +20,7 @@ import { Target, TargetsResult, } from "../../api/responseTypes/targets"; -import { FC, useMemo } from "react"; +import { FC, Suspense, useMemo, useState } from "react"; import { useAppDispatch, useAppSelector } from "../../state/hooks"; import { setCollapsedPools, @@ -28,6 +31,8 @@ import CustomInfiniteScroll from "../../components/CustomInfiniteScroll"; import { useDebouncedValue, useLocalStorage } from "@mantine/hooks"; import { targetPoolDisplayLimit } from "./ServiceDiscoveryPage"; import { LabelBadges } from "../../components/LabelBadges"; +import ErrorBoundary from "../../components/ErrorBoundary"; +import RelabelSteps from "./RelabelSteps"; type TargetLabels = { discoveredLabels: Labels; @@ -162,6 +167,10 @@ const ScrapePoolList: FC = ({ key: "serviceDiscoveryPage.showEmptyPools", defaultValue: false, }); + const [showRelabelingSteps, setShowRelabelingSteps] = useState<{ + labels: Labels; + pool: string; + } | null>(null); // Based on the selected pool (if any), load the list of targets. const { @@ -333,16 +342,29 @@ const ScrapePoolList: FC = ({ py="lg" valign={target.isDropped ? "middle" : "top"} > - {target.isDropped ? ( - - dropped due to relabeling rules - - ) : ( - - )} + + {target.isDropped ? ( + + dropped + + ) : ( + + )} + { + setShowRelabelingSteps({ + labels: target.discoveredLabels, + pool: poolName, + }); + }} + > + show relabeling + + ))} @@ -356,6 +378,32 @@ const ScrapePoolList: FC = ({ ); })} + setShowRelabelingSteps(null)} + title="Relabeling steps for target" + > + + + {Array.from(Array(20), (_, i) => ( + + ))} + + } + > + + + + ); };