mirrors/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2025-12-01 15:41:12 +01:00
Code Issues Projects Releases Wiki Activity

ui: Allow viewing detailed relabeling steps for each discovered target

Browse Source 
This adds:

* A `ScrapePoolConfig()` method to the scrape manager that allows getting
  the scrape config for a given pool.
* An API endpoint at `/api/v1/targets/relabel_steps` that takes a pool name
  and a label set of a target and returns a detailed list of applied
  relabeling rules and their output for each step.
* A "show relabeling" link/button for each target on the discovery page
  that shows the detailed flow of all relabeling rules (based on the API
  response) for that target.

Note that this changes the JSON encoding of the relabeling rule config
struct to output the original snake_case (instead of camelCase) field names,
and before merging, we need to be sure that's ok :) See my comment about
that at https://github.com/prometheus/prometheus/pull/15383#issuecomment-3405591487

Fixes https://github.com/prometheus/prometheus/issues/17283

Signed-off-by: Julius Volz <julius.volz@gmail.com>
This commit is contained in:
Julius Volz 2025-10-15 15:08:07 +02:00
parent fd421dc3b1
commit 8b1bd7d6c3
9 changed files with 439 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
docs/querying/api.md
View File

@ -810,6 +810,64 @@ curl 'http://localhost:9090/api/v1/targets?scrapePool=node_exporter'
}
```
## Relabel steps
This endpoint is **experimental** and might change in the future. It is currently only meant to be used by Prometheus' own web UI, and the endpoint name and exact format returned may change from one Prometheus version to another. It may also be removed again in case it is no longer needed by the UI.
The following endpoint returns a step-by-step list of relabeling rules and their effects on a given target's label set.
```
GET /api/v1/targets/relabel_steps
```
URL query parameters:
- `scrapePool=<string>`: The scrape pool name of the target, used to determine the relabeling rules to apply. Required.
- `labels=<string>`: A JSON object containing the label set of the target before any relabeling is applied. Required.
The following example returns the relabeling steps for a discovered target in the `prometheus` scrape pool with the label set `{"__address__": "localhost:9090", "job": "prometheus"}`:
```bash
curl -g 'http://localhost:9090/api/v1/targets/relabel_steps?scrapePool=prometheus&labels={"__address__":"localhost:9090","job":"prometheus"}'
```
```json
{
"data" : {
"steps" : [
{
"keep" : true,
"output" : {
"__address__" : "localhost:9090",
"env" : "development",
"job" : "prometheus"
},
"rule" : {
"action" : "replace",
"regex" : "(.*)",
"replacement" : "development",
"separator" : ";",
"target_label" : "env"
}
},
{
"keep" : false,
"output" : {},
"rule" : {
"action" : "drop",
"regex" : "localhost:.*",
"replacement" : "$1",
"separator" : ";",
"source_labels" : [
"__address__"
]
}
}
]
},
"status" : "success"
}
```
## Rules
The `/rules` API endpoint returns a list of alerting and recording rules that

4
model/relabel/relabel.go
View File

@ -86,7 +86,7 @@ func (a *Action) UnmarshalYAML(unmarshal func(any) error) error {
type Config struct {
// A list of labels from which values are taken and concatenated
// with the configured separator in order.
SourceLabels model.LabelNames `yaml:"source_labels,flow,omitempty" json:"sourceLabels,omitempty"`
SourceLabels model.LabelNames `yaml:"source_labels,flow,omitempty" json:"source_labels,omitempty"`
// Separator is the string between concatenated values from the source labels.
Separator string `yaml:"separator,omitempty" json:"separator,omitempty"`
// Regex against which the concatenation is matched.
@ -95,7 +95,7 @@ type Config struct {
Modulus uint64 `yaml:"modulus,omitempty" json:"modulus,omitempty"`
// TargetLabel is the label to which the resulting string is written in a replacement.
// Regexp interpolation is allowed for the replace action.
TargetLabel string `yaml:"target_label,omitempty" json:"targetLabel,omitempty"`
TargetLabel string `yaml:"target_label,omitempty" json:"target_label,omitempty"`
// Replacement is the regex replacement pattern to be used.
Replacement string `yaml:"replacement,omitempty" json:"replacement,omitempty"`
// Action is the action to be performed for the relabeling.

12
scrape/manager.go
View File

@ -399,3 +399,15 @@ func (m *Manager) TargetsDroppedCounts() map[string]int {
}
return counts
}
func (m *Manager) ScrapePoolConfig(scrapePool string) (*config.ScrapeConfig, error) {
m.mtxScrape.Lock()
defer m.mtxScrape.Unlock()
sp, ok := m.scrapePools[scrapePool]
if !ok {
return nil, fmt.Errorf("scrape pool %q not found", scrapePool)
}
return sp.config, nil
}

46
web/api/v1/api.go
View File

@ -44,6 +44,7 @@ import (
"github.com/prometheus/prometheus/config"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/model/metadata"
"github.com/prometheus/prometheus/model/relabel"
"github.com/prometheus/prometheus/model/timestamp"
"github.com/prometheus/prometheus/promql"
"github.com/prometheus/prometheus/promql/parser"
@ -129,6 +130,7 @@ type TargetRetriever interface {
TargetsActive() map[string][]*scrape.Target
TargetsDropped() map[string][]*scrape.Target
TargetsDroppedCounts() map[string]int
ScrapePoolConfig(string) (*config.ScrapeConfig, error)
}
// AlertmanagerRetriever provides a list of all/dropped AlertManager URLs.
@ -429,6 +431,7 @@ func (api *API) Register(r *route.Router) {
r.Get("/scrape_pools", wrap(api.scrapePools))
r.Get("/targets", wrap(api.targets))
r.Get("/targets/metadata", wrap(api.targetMetadata))
r.Get("/targets/relabel_steps", wrap(api.targetRelabelSteps))
r.Get("/alertmanagers", wrapAgent(api.alertmanagers))
r.Get("/metadata", wrap(api.metricMetadata))
@ -1303,6 +1306,49 @@ type metricMetadata struct {
Unit string `json:"unit"`
}
type RelabelStep struct {
Rule *relabel.Config `json:"rule"`
Output labels.Labels `json:"output"`
Keep bool `json:"keep"`
}
type RelabelStepsResponse struct {
Steps []RelabelStep `json:"steps"`
}
func (api *API) targetRelabelSteps(r *http.Request) apiFuncResult {
scrapePool := r.FormValue("scrapePool")
if scrapePool == "" {
return apiFuncResult{nil, &apiError{errorBadData, errors.New("no scrapePool parameter provided")}, nil, nil}
}
labelsJSON := r.FormValue("labels")
if labelsJSON == "" {
return apiFuncResult{nil, &apiError{errorBadData, errors.New("no labels parameter provided")}, nil, nil}
}
var lbls labels.Labels
if err := json.Unmarshal([]byte(labelsJSON), &lbls); err != nil {
return apiFuncResult{nil, &apiError{errorBadData, fmt.Errorf("error parsing labels: %w", err)}, nil, nil}
}
scrapeConfig, err := api.targetRetriever(r.Context()).ScrapePoolConfig(scrapePool)
if err != nil {
return apiFuncResult{nil, &apiError{errorBadData, fmt.Errorf("error retrieving scrape config: %w", err)}, nil, nil}
}
rules := scrapeConfig.RelabelConfigs
steps := make([]RelabelStep, len(rules))
for i, rule := range rules {
outLabels, keep := relabel.Process(lbls, rules[:i+1]...)
steps[i] = RelabelStep{
Rule: rule,
Output: outLabels,
Keep: keep,
}
}
return apiFuncResult{&RelabelStepsResponse{Steps: steps}, nil, nil, nil}
}
// AlertmanagerDiscovery has all the active Alertmanagers.
type AlertmanagerDiscovery struct {
ActiveAlertmanagers []*AlertmanagerTarget `json:"activeAlertmanagers"`

58
web/api/v1/api_test.go
View File

@ -31,6 +31,9 @@ import (
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/grafana/regexp"
jsoniter "github.com/json-iterator/go"
"github.com/oklog/ulid/v2"
"github.com/prometheus/client_golang/prometheus"
@ -44,6 +47,7 @@ import (
"github.com/prometheus/prometheus/model/exemplar"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/model/metadata"
"github.com/prometheus/prometheus/model/relabel"
"github.com/prometheus/prometheus/model/timestamp"
"github.com/prometheus/prometheus/prompb"
"github.com/prometheus/prometheus/promql"
@ -162,6 +166,25 @@ func (t testTargetRetriever) TargetsDroppedCounts() map[string]int {
return r
}
func (testTargetRetriever) ScrapePoolConfig(_ string) (*config.ScrapeConfig, error) {
return &config.ScrapeConfig{
RelabelConfigs: []*relabel.Config{
{
Action: relabel.Replace,
Replacement: "example.com:443",
TargetLabel: "__address__",
Regex: relabel.MustNewRegexp(""),
NameValidationScheme: model.LegacyValidation,
},
{
Action: relabel.Drop,
SourceLabels: []model.LabelName{"__address__"},
Regex: relabel.MustNewRegexp(`example\.com:.*`),
},
},
}, nil
}
func (t *testTargetRetriever) SetMetadataStoreForTargets(identifier string, metadata scrape.MetricMetadataStore) error {
targets, ok := t.activeTargets[identifier]
@ -1883,6 +1906,37 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E
DroppedTargetCounts: map[string]int{"blackbox": 1},
},
},
{
endpoint: api.targetRelabelSteps,
query: url.Values{"scrapePool": []string{"testpool"}, "labels": []string{`{"job":"test","__address__":"localhost:9090"}`}},
response: &RelabelStepsResponse{
Steps: []RelabelStep{
{
Rule: &relabel.Config{
Action: relabel.Replace,
Replacement: "example.com:443",
TargetLabel: "__address__",
Regex: relabel.MustNewRegexp(""),
NameValidationScheme: model.LegacyValidation,
},
Output: labels.FromMap(map[string]string{
"job": "test",
"__address__": "example.com:443",
}),
Keep: true,
},
{
Rule: &relabel.Config{
Action: relabel.Drop,
SourceLabels: []model.LabelName{"__address__"},
Regex: relabel.MustNewRegexp(`example\.com:.*`),
},
Output: labels.EmptyLabels(),
Keep: false,
},
},
},
},
// With a matching metric.
{
endpoint: api.targetMetadata,
@ -3772,7 +3826,9 @@ func assertAPIError(t *testing.T, got *apiError, exp errorType) {
func assertAPIResponse(t *testing.T, got, exp any) {
t.Helper()
testutil.RequireEqual(t, exp, got)
testutil.RequireEqualWithOptions(t, exp, got, []cmp.Option{
cmpopts.IgnoreUnexported(regexp.Regexp{}),
})
}
func assertAPIResponseLength(t *testing.T, got any, expLen int) {

4
web/api/v1/errors_test.go
View File

@ -264,6 +264,10 @@ func (DummyTargetRetriever) TargetsDroppedCounts() map[string]int {
return nil
}
func (DummyTargetRetriever) ScrapePoolConfig(_ string) (*config.ScrapeConfig, error) {
return nil, errors.New("not implemented")
}
// DummyAlertmanagerRetriever implements AlertmanagerRetriever.
type DummyAlertmanagerRetriever struct{}

13
web/ui/mantine-ui/src/api/responseTypes/relabel_steps.ts Normal file
View File

@ -0,0 +1,13 @@
import { Labels } from "./targets";
export type RelabelStep = {
rule: { [key: string]: unknown };
output: Labels;
keep: boolean;
};
// Result type for /api/v1/relabel_steps endpoint.
// See: https://prometheus.io/docs/prometheus/latest/querying/api/#relabel_steps
export type RelabelStepsResult = {
steps: RelabelStep[];
};

188
web/ui/mantine-ui/src/pages/service-discovery/RelabelSteps.tsx Normal file
View File

@ -0,0 +1,188 @@
import { em, Group, Stack, Table, Text } from "@mantine/core";
import { useSuspenseAPIQuery } from "../../api/api";
import { RelabelStepsResult } from "../../api/responseTypes/relabel_steps";
import { Labels } from "../../api/responseTypes/targets";
import React from "react";
import {
IconArrowDown,
IconCircleMinus,
IconCirclePlus,
IconReplace,
IconTags,
} from "@tabler/icons-react";
const iconStyle = { width: em(18), height: em(18), verticalAlign: "middle" };
const ruleTable = (rule: { [key: string]: unknown }, idx: number) => {
return (
<Table
w="60%"
withTableBorder
withColumnBorders
bg="light-dark(var(--mantine-color-gray-0), var(--mantine-color-gray-8))"
>
<Table.Thead>
<Table.Tr
bg={
"light-dark(var(--mantine-color-gray-1), var(--mantine-color-gray-7))"
}
>
<Table.Th colSpan={2}>
<Group gap="xs">
<IconReplace style={iconStyle} /> Rule {idx + 1}
</Group>
</Table.Th>
</Table.Tr>
</Table.Thead>
<Table.Tbody>
{Object.entries(rule)
.sort(([a], [b]) => {
// Sort by a predefined order for known fields, otherwise alphabetically.
const sortedRuleFieldNames: string[] = [
"action",
"source_labels",
"regex",
"modulus",
"replacement",
"target_label",
];
const ai = sortedRuleFieldNames.indexOf(a);
const bi = sortedRuleFieldNames.indexOf(b);
if (ai === -1 && bi === -1) {
return a.localeCompare(b);
}
if (ai === -1) {
return 1;
}
if (bi === -1) {
return -1;
}
return ai - bi;
})
.map(([k, v]) => (
<Table.Tr key={k}>
<Table.Th>
<code>{k}</code>
</Table.Th>
<Table.Td>
<code>
{typeof v === "object" ? JSON.stringify(v) : String(v)}
</code>
</Table.Td>
</Table.Tr>
))}
</Table.Tbody>
</Table>
);
};
const labelsTable = (labels: Labels, prevLabels: Labels | null) => {
if (labels === null) {
return <Text c="dimmed">dropped</Text>;
}
return (
<Table w="50%" withTableBorder>
<Table.Thead>
<Table.Tr
bg={
"light-dark(var(--mantine-color-gray-1), var(--mantine-color-gray-7))"
}
>
<Table.Th colSpan={3}>
<Group gap="xs">
<IconTags style={iconStyle} /> Labels
</Group>
</Table.Th>
</Table.Tr>
</Table.Thead>
<Table.Tbody>
{Object.entries(labels)
.concat(
prevLabels
? Object.entries(prevLabels).filter(
([k]) => labels[k] === undefined
)
: []
)
.sort(([a], [b]) => a.localeCompare(b))
.map(([k, v]) => {
const added = prevLabels !== null && prevLabels[k] === undefined;
const changed =
prevLabels !== null && !added && prevLabels[k] !== v;
const removed =
prevLabels !== null &&
!changed &&
prevLabels[k] !== undefined &&
labels[k] === undefined;
return (
<Table.Tr
key={k}
bg={
added
? "light-dark(var(--mantine-color-green-1), var(--mantine-color-green-8))"
: changed
? "light-dark(var(--mantine-color-blue-1), var(--mantine-color-blue-8))"
: removed
? "light-dark(var(--mantine-color-red-1), var(--mantine-color-red-8))"
: undefined
}
>
<Table.Td w={40}>
{added ? (
<IconCirclePlus style={iconStyle} />
) : changed ? (
<IconReplace style={iconStyle} />
) : removed ? (
<IconCircleMinus style={iconStyle} />
) : null}
</Table.Td>
<Table.Th>
<code>{k}</code>
</Table.Th>
<Table.Td>
<code>{v}</code>
</Table.Td>
</Table.Tr>
);
})}
</Table.Tbody>
</Table>
);
};
const RelabelSteps: React.FC<{
labels: Labels;
pool: string;
}> = ({ labels, pool }) => {
const { data } = useSuspenseAPIQuery<RelabelStepsResult>({
path: `/targets/relabel_steps`,
params: {
labels: JSON.stringify(labels),
scrapePool: pool,
},
});
return (
<Stack align="center">
{labelsTable(labels, null)}
{data.data.steps.map((step, idx) => (
<React.Fragment key={idx}>
<IconArrowDown style={iconStyle} />
{ruleTable(step.rule, idx)}
<IconArrowDown style={iconStyle} />
{step.keep ? (
labelsTable(
step.output,
idx === 0 ? labels : data.data.steps[idx - 1].output
)
) : (
<Text c="dimmed">dropped</Text>
)}
</React.Fragment>
))}
</Stack>
);
};
export default RelabelSteps;

70
web/ui/mantine-ui/src/pages/service-discovery/ServiceDiscoveryPoolsList.tsx
View File

@ -2,8 +2,11 @@ import {
Accordion,
Alert,
Anchor,
Box,
Group,
Modal,
RingProgress,
Skeleton,
Stack,
Table,
Text,
@ -17,7 +20,7 @@ import {
Target,
TargetsResult,
} from "../../api/responseTypes/targets";
import { FC, useMemo } from "react";
import { FC, Suspense, useMemo, useState } from "react";
import { useAppDispatch, useAppSelector } from "../../state/hooks";
import {
setCollapsedPools,
@ -28,6 +31,8 @@ import CustomInfiniteScroll from "../../components/CustomInfiniteScroll";
import { useDebouncedValue, useLocalStorage } from "@mantine/hooks";
import { targetPoolDisplayLimit } from "./ServiceDiscoveryPage";
import { LabelBadges } from "../../components/LabelBadges";
import ErrorBoundary from "../../components/ErrorBoundary";
import RelabelSteps from "./RelabelSteps";
type TargetLabels = {
discoveredLabels: Labels;
@ -162,6 +167,10 @@ const ScrapePoolList: FC<ScrapePoolListProp> = ({
key: "serviceDiscoveryPage.showEmptyPools",
defaultValue: false,
});
const [showRelabelingSteps, setShowRelabelingSteps] = useState<{
labels: Labels;
pool: string;
} | null>(null);
// Based on the selected pool (if any), load the list of targets.
const {
@ -333,16 +342,29 @@ const ScrapePoolList: FC<ScrapePoolListProp> = ({
py="lg"
valign={target.isDropped ? "middle" : "top"}
>
{target.isDropped ? (
<Text c="blue.6" fw="bold">
dropped due to relabeling rules
</Text>
) : (
<LabelBadges
labels={target.labels}
wrapper={Stack}
/>
)}
<Stack>
{target.isDropped ? (
<Text c="dimmed" fw="bold">
dropped
</Text>
) : (
<LabelBadges
labels={target.labels}
wrapper={Stack}
/>
)}
<Anchor
inherit
onClick={() => {
setShowRelabelingSteps({
labels: target.discoveredLabels,
pool: poolName,
});
}}
>
show relabeling
</Anchor>
</Stack>
</Table.Td>
</Table.Tr>
))}
@ -356,6 +378,32 @@ const ScrapePoolList: FC<ScrapePoolListProp> = ({
);
})}
</Accordion>
<Modal
size="95%"
opened={showRelabelingSteps !== null}
onClose={() => setShowRelabelingSteps(null)}
title="Relabeling steps for target"
>
<ErrorBoundary
key={location.pathname}
title="Error showing relabeling steps"
>
<Suspense
fallback={
<Box mt="lg">
{Array.from(Array(20), (_, i) => (
<Skeleton key={i} height={30} mb={15} width="100%" />
))}
</Box>
}
>
<RelabelSteps
pool={showRelabelingSteps?.pool || ""}
labels={showRelabelingSteps?.labels || {}}
/>
</Suspense>
</ErrorBoundary>
</Modal>
</Stack>
);
};