diff --git a/docs/querying/api.md b/docs/querying/api.md index a45e3c292b..111a4e205f 100644 --- a/docs/querying/api.md +++ b/docs/querying/api.md @@ -1562,6 +1562,65 @@ NOTE: This endpoint is available before the server has been marked ready and is *New in v2.28* +### Self Metrics + +**NOTE**: This endpoint is **experimental** and might change in the future. + +The following endpoint returns Prometheus' own instrumentation metrics from its internal client registry as structured JSON. These are the same metrics that are exposed on the `/metrics` endpoint in Prometheus text exposition format, but returned as JSON for programmatic access by the web UI. + +The response uses the standard [ProtoJSON](https://protobuf.dev/programming-guides/json/) representation of the `io.prometheus.client.MetricFamily` protocol buffer message. + +``` +GET /api/v1/status/self_metrics +``` + +URL query parameters: + +- `metric_name_pattern=`: A regular expression filter for metric names (fully anchored, like PromQL label matchers). Only metric families whose names fully match the pattern are returned. For example, `metric_name_pattern=prometheus_tsdb_.*` returns all metric families whose names start with `prometheus_tsdb_`. Optional. When omitted, all metric families are returned. + +Each returned metric family is a ProtoJSON-encoded `MetricFamily` containing: + +- **name**: The metric name. +- **help**: The metric help string. +- **type**: The metric type (`COUNTER`, `GAUGE`, `SUMMARY`, `HISTOGRAM`, `UNTYPED`). +- **unit**: The metric unit, if set (optional). +- **metric**: A list of individual metrics, each containing: + - **label**: A list of `{name, value}` label pairs. + - **gauge**, **counter**, **summary**, **histogram**, or **untyped**: The type-specific metric data. + +```bash +curl 'http://localhost:9090/api/v1/status/self_metrics?metric_name_pattern=prometheus_build_info' +``` + +```json +{ + "status": "success", + "data": [ + { + "name": "prometheus_build_info", + "help": "A metric with a constant '1' value labeled by version, revision, branch, goversion from which prometheus was built, and the goos and goarch for the build.", + "type": "GAUGE", + "metric": [ + { + "label": [ + { "name": "branch", "value": "main" }, + { "name": "goarch", "value": "amd64" }, + { "name": "goos", "value": "linux" }, + { "name": "goversion", "value": "go1.26.1-X:nodwarf5" }, + { "name": "revision", "value": "7b5a4090e38d9e1ad7697c7641234f4ed135a6c7" }, + { "name": "tags", "value": "netgo,builtinassets" }, + { "name": "version", "value": "3.11.0-rc.0" } + ], + "gauge": { + "value": 1 + } + } + ] + } + ] +} +``` + ## TSDB Admin APIs These are APIs that expose database functionalities for the advanced user. These APIs are not enabled unless the `--web.enable-admin-api` is set. diff --git a/web/api/v1/api.go b/web/api/v1/api.go index c236f402d3..28ad09051e 100644 --- a/web/api/v1/api.go +++ b/web/api/v1/api.go @@ -41,6 +41,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" "github.com/prometheus/common/route" + "google.golang.org/protobuf/encoding/protojson" "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/model/labels" @@ -458,6 +459,7 @@ func (api *API) Register(r *route.Router) { r.Get("/status/flags", wrap(api.serveFlags)) r.Get("/status/tsdb", wrapAgent(api.serveTSDBStatus)) r.Get("/status/tsdb/blocks", wrapAgent(api.serveTSDBBlocks)) + r.Get("/status/self_metrics", wrap(api.selfMetrics)) r.Get("/features", wrap(api.features)) r.Get("/status/walreplay", api.serveWALReplayStatus) r.Get("/notifications", api.notifications) @@ -1865,6 +1867,38 @@ func TSDBStatsFromIndexStats(stats []index.Stat) []TSDBStat { return result } +func (api *API) selfMetrics(r *http.Request) apiFuncResult { + var nameFilter *regexp.Regexp + if pattern := r.FormValue("metric_name_pattern"); pattern != "" { + var err error + nameFilter, err = regexp.Compile("^(?:" + pattern + ")$") + if err != nil { + return apiFuncResult{nil, &apiError{errorBadData, fmt.Errorf("invalid metric_name_pattern: %w", err)}, nil, nil} + } + } + + mfs, err := api.gatherer.Gather() + if err != nil { + return apiFuncResult{nil, &apiError{errorInternal, fmt.Errorf("error gathering self metrics: %w", err)}, nil, nil} + } + + marshaler := protojson.MarshalOptions{} + result := make([]json.RawMessage, 0, len(mfs)) + for _, mf := range mfs { + if nameFilter != nil && !nameFilter.MatchString(mf.GetName()) { + continue + } + + b, err := marshaler.Marshal(mf) + if err != nil { + return apiFuncResult{nil, &apiError{errorInternal, fmt.Errorf("error marshaling metric family %q: %w", mf.GetName(), err)}, nil, nil} + } + result = append(result, json.RawMessage(b)) + } + + return apiFuncResult{result, nil, nil, nil} +} + func (api *API) serveTSDBBlocks(*http.Request) apiFuncResult { blockMetas, err := api.db.BlockMetas() if err != nil { diff --git a/web/api/v1/api_test.go b/web/api/v1/api_test.go index 9b5b0af37f..e4a3dc59d7 100644 --- a/web/api/v1/api_test.go +++ b/web/api/v1/api_test.go @@ -4550,6 +4550,74 @@ func TestTSDBStatus(t *testing.T) { } } +func TestSelfMetrics(t *testing.T) { + api := &API{gatherer: prometheus.DefaultGatherer} + + t.Run("returns all metrics without filter", func(t *testing.T) { + req, err := http.NewRequest(http.MethodGet, "", http.NoBody) + require.NoError(t, err) + res := api.selfMetrics(req) + assertAPIError(t, res.err, errorNone) + families, ok := res.data.([]json.RawMessage) + require.True(t, ok, "expected []json.RawMessage") + require.NotEmpty(t, families, "expected at least one metric family from default gatherer") + }) + + t.Run("filters metrics by regex", func(t *testing.T) { + req, err := http.NewRequest(http.MethodGet, "?metric_name_pattern=go_.*", http.NoBody) + require.NoError(t, err) + req.Form = url.Values{"metric_name_pattern": {"go_.*"}} + res := api.selfMetrics(req) + assertAPIError(t, res.err, errorNone) + families, ok := res.data.([]json.RawMessage) + require.True(t, ok, "expected []json.RawMessage") + require.NotEmpty(t, families) + for _, raw := range families { + var f map[string]any + require.NoError(t, json.Unmarshal(raw, &f)) + name, _ := f["name"].(string) + require.True(t, strings.HasPrefix(name, "go_"), "expected metric name to start with 'go_', got %q", name) + } + }) + + t.Run("non-matching pattern returns empty", func(t *testing.T) { + req, err := http.NewRequest(http.MethodGet, "?metric_name_pattern=nonexistent_prefix_xyz_", http.NoBody) + require.NoError(t, err) + req.Form = url.Values{"metric_name_pattern": {"nonexistent_prefix_xyz_"}} + res := api.selfMetrics(req) + assertAPIError(t, res.err, errorNone) + families, ok := res.data.([]json.RawMessage) + require.True(t, ok, "expected []json.RawMessage") + require.Empty(t, families) + }) + + t.Run("invalid regex returns error", func(t *testing.T) { + req, err := http.NewRequest(http.MethodGet, "?metric_name_pattern=[invalid", http.NoBody) + require.NoError(t, err) + req.Form = url.Values{"metric_name_pattern": {"[invalid"}} + res := api.selfMetrics(req) + assertAPIError(t, res.err, errorBadData) + }) + + t.Run("metric families have expected ProtoJSON structure", func(t *testing.T) { + req, err := http.NewRequest(http.MethodGet, "", http.NoBody) + require.NoError(t, err) + res := api.selfMetrics(req) + assertAPIError(t, res.err, errorNone) + families := res.data.([]json.RawMessage) + for _, raw := range families { + var f map[string]any + require.NoError(t, json.Unmarshal(raw, &f)) + name, _ := f["name"].(string) + require.NotEmpty(t, name, "metric family name should not be empty") + typ, _ := f["type"].(string) + require.NotEmpty(t, typ, "metric family type should not be empty") + metrics, _ := f["metric"].([]any) + require.NotEmpty(t, metrics, "metric family %q should have at least one metric", name) + } + }) +} + func TestReturnAPIError(t *testing.T) { cases := []struct { err error diff --git a/web/api/v1/openapi.go b/web/api/v1/openapi.go index 59fa8969ef..839af8108e 100644 --- a/web/api/v1/openapi.go +++ b/web/api/v1/openapi.go @@ -298,6 +298,7 @@ func (b *OpenAPIBuilder) getAllPathDefinitions() *orderedmap.Map[string, *v3.Pat paths.Set("/status/tsdb", b.statusTSDBPath()) paths.Set("/status/tsdb/blocks", b.statusTSDBBlocksPath()) paths.Set("/status/walreplay", b.statusWALReplayPath()) + paths.Set("/status/self_metrics", b.statusSelfMetricsPath()) // Admin endpoints. paths.Set("/admin/tsdb/delete_series", b.adminDeleteSeriesPath()) diff --git a/web/api/v1/openapi_examples.go b/web/api/v1/openapi_examples.go index a1db5d5f3b..25637adcde 100644 --- a/web/api/v1/openapi_examples.go +++ b/web/api/v1/openapi_examples.go @@ -886,6 +886,55 @@ func statusWALReplayResponseExamples() *orderedmap.Map[string, *base.Example] { return examples } +// statusSelfMetricsResponseExamples returns examples for /status/self_metrics response. +func statusSelfMetricsResponseExamples() *orderedmap.Map[string, *base.Example] { + examples := orderedmap.New[string, *base.Example]() + + examples.Set("selfMetrics", &base.Example{ + Summary: "Prometheus self-instrumentation metrics in ProtoJSON format", + Value: createYAMLNode(map[string]any{ + "status": "success", + "data": []map[string]any{ + { + "name": "prometheus_build_info", + "help": "A metric with a constant '1' value labeled by version, revision, branch, goversion from which prometheus was built, and the goos and goarch for the build.", + "type": "GAUGE", + "metric": []map[string]any{ + { + "label": []map[string]string{ + {"name": "branch", "value": "HEAD"}, + {"name": "goarch", "value": "amd64"}, + {"name": "goos", "value": "linux"}, + {"name": "goversion", "value": "go1.23.0"}, + {"name": "revision", "value": "abc1234"}, + {"name": "tags", "value": "netgo,builtinassets,stringlabels"}, + {"name": "version", "value": "3.0.0"}, + }, + "gauge": map[string]any{ + "value": 1, + }, + }, + }, + }, + { + "name": "prometheus_tsdb_head_chunks", + "help": "Total number of chunks in the head block.", + "type": "GAUGE", + "metric": []map[string]any{ + { + "gauge": map[string]any{ + "value": 1024, + }, + }, + }, + }, + }, + }), + }) + + return examples +} + // deleteSeriesResponseExamples returns examples for /admin/tsdb/delete_series response. func deleteSeriesResponseExamples() *orderedmap.Map[string, *base.Example] { examples := orderedmap.New[string, *base.Example]() diff --git a/web/api/v1/openapi_paths.go b/web/api/v1/openapi_paths.go index b2622f4ff0..6aa4310637 100644 --- a/web/api/v1/openapi_paths.go +++ b/web/api/v1/openapi_paths.go @@ -441,6 +441,22 @@ func (*OpenAPIBuilder) statusWALReplayPath() *v3.PathItem { } } +func (*OpenAPIBuilder) statusSelfMetricsPath() *v3.PathItem { + params := []*v3.Parameter{ + queryParamWithExample("metric_name_pattern", "Regular expression filter for metric names (fully anchored, like PromQL label matchers). Only metric families whose names fully match the pattern are returned.", false, stringSchema(), []example{{"example", "prometheus_tsdb_.*"}}), + } + return &v3.PathItem{ + Get: &v3.Operation{ + OperationId: "get-status-self-metrics", + Summary: "Get Prometheus self-instrumentation metrics", + Description: "Returns Prometheus' own instrumentation metrics from its internal client registry, as structured JSON. Supports optional regex filtering via the metric_name_pattern parameter.", + Tags: []string{"status"}, + Parameters: params, + Responses: responsesWithErrorExamples("StatusSelfMetricsOutputBody", statusSelfMetricsResponseExamples(), errorResponseExamples(), "Self metrics retrieved successfully.", "Error retrieving self metrics."), + }, + } +} + func (*OpenAPIBuilder) adminDeleteSeriesPath() *v3.PathItem { params := []*v3.Parameter{ queryParamWithExample("match[]", "Series selectors to identify series to delete.", true, base.CreateSchemaProxy(&base.Schema{ diff --git a/web/api/v1/openapi_schemas.go b/web/api/v1/openapi_schemas.go index ac298ac0bd..d768f4208d 100644 --- a/web/api/v1/openapi_schemas.go +++ b/web/api/v1/openapi_schemas.go @@ -112,6 +112,7 @@ func (b *OpenAPIBuilder) buildComponents() *v3.Components { schemas.Set("StatusTSDBBlocksOutputBody", b.refResponseBodySchema("StatusTSDBBlocksData", "Response body for status TSDB blocks endpoint.")) schemas.Set("StatusWALReplayData", b.statusWALReplayDataSchema()) schemas.Set("StatusWALReplayOutputBody", b.refResponseBodySchema("StatusWALReplayData", "Response body for status WAL replay endpoint.")) + schemas.Set("StatusSelfMetricsOutputBody", b.simpleResponseBodySchema()) // Admin schemas. schemas.Set("DeleteSeriesOutputBody", b.statusOnlyResponseBodySchema()) diff --git a/web/api/v1/testdata/openapi_3.1_golden.yaml b/web/api/v1/testdata/openapi_3.1_golden.yaml index 47f3d06621..8d1278ff91 100644 --- a/web/api/v1/testdata/openapi_3.1_golden.yaml +++ b/web/api/v1/testdata/openapi_3.1_golden.yaml @@ -2074,6 +2074,77 @@ paths: error: TSDB not ready errorType: internal status: error + /status/self_metrics: + get: + tags: + - status + summary: Get Prometheus self-instrumentation metrics + description: Returns Prometheus' own instrumentation metrics from its internal client registry, as structured JSON. Supports optional regex filtering via the metric_name_pattern parameter. + operationId: get-status-self-metrics + parameters: + - name: metric_name_pattern + in: query + description: Regular expression filter for metric names (fully anchored, like PromQL label matchers). Only metric families whose names fully match the pattern are returned. + required: false + explode: false + schema: + type: string + examples: + example: + value: prometheus_tsdb_.* + responses: + "200": + description: Self metrics retrieved successfully. + content: + application/json: + schema: + $ref: '#/components/schemas/StatusSelfMetricsOutputBody' + examples: + selfMetrics: + summary: Prometheus self-instrumentation metrics in ProtoJSON format + value: + data: + - help: A metric with a constant '1' value labeled by version, revision, branch, goversion from which prometheus was built, and the goos and goarch for the build. + metric: + - gauge: + value: 1 + label: + - name: branch + value: HEAD + - name: goarch + value: amd64 + - name: goos + value: linux + - name: goversion + value: go1.23.0 + - name: revision + value: abc1234 + - name: tags + value: netgo,builtinassets,stringlabels + - name: version + value: 3.0.0 + name: prometheus_build_info + type: GAUGE + - help: Total number of chunks in the head block. + metric: + - gauge: + value: 1024 + name: prometheus_tsdb_head_chunks + type: GAUGE + status: success + default: + description: Error retrieving self metrics. + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + examples: + tsdbNotReady: + summary: TSDB not ready + value: + error: TSDB not ready + errorType: internal + status: error /admin/tsdb/delete_series: put: tags: @@ -4202,6 +4273,35 @@ components: - data additionalProperties: false description: Response body for status WAL replay endpoint. + StatusSelfMetricsOutputBody: + type: object + properties: + status: + type: string + enum: + - success + - error + description: Response status. + example: success + data: + description: Response data (structure varies by endpoint). + example: + result: ok + warnings: + type: array + items: + type: string + description: Only set if there were warnings while executing the request. There will still be data in the data field. + infos: + type: array + items: + type: string + description: Only set if there were info-level annotations while executing the request. + required: + - status + - data + additionalProperties: false + description: Generic response body. DeleteSeriesOutputBody: type: object properties: diff --git a/web/api/v1/testdata/openapi_3.2_golden.yaml b/web/api/v1/testdata/openapi_3.2_golden.yaml index 7766f86f39..a3f3d88f52 100644 --- a/web/api/v1/testdata/openapi_3.2_golden.yaml +++ b/web/api/v1/testdata/openapi_3.2_golden.yaml @@ -2074,6 +2074,77 @@ paths: error: TSDB not ready errorType: internal status: error + /status/self_metrics: + get: + tags: + - status + summary: Get Prometheus self-instrumentation metrics + description: Returns Prometheus' own instrumentation metrics from its internal client registry, as structured JSON. Supports optional regex filtering via the metric_name_pattern parameter. + operationId: get-status-self-metrics + parameters: + - name: metric_name_pattern + in: query + description: Regular expression filter for metric names (fully anchored, like PromQL label matchers). Only metric families whose names fully match the pattern are returned. + required: false + explode: false + schema: + type: string + examples: + example: + value: prometheus_tsdb_.* + responses: + "200": + description: Self metrics retrieved successfully. + content: + application/json: + schema: + $ref: '#/components/schemas/StatusSelfMetricsOutputBody' + examples: + selfMetrics: + summary: Prometheus self-instrumentation metrics in ProtoJSON format + value: + data: + - help: A metric with a constant '1' value labeled by version, revision, branch, goversion from which prometheus was built, and the goos and goarch for the build. + metric: + - gauge: + value: 1 + label: + - name: branch + value: HEAD + - name: goarch + value: amd64 + - name: goos + value: linux + - name: goversion + value: go1.23.0 + - name: revision + value: abc1234 + - name: tags + value: netgo,builtinassets,stringlabels + - name: version + value: 3.0.0 + name: prometheus_build_info + type: GAUGE + - help: Total number of chunks in the head block. + metric: + - gauge: + value: 1024 + name: prometheus_tsdb_head_chunks + type: GAUGE + status: success + default: + description: Error retrieving self metrics. + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + examples: + tsdbNotReady: + summary: TSDB not ready + value: + error: TSDB not ready + errorType: internal + status: error /admin/tsdb/delete_series: put: tags: @@ -4240,6 +4311,35 @@ components: - data additionalProperties: false description: Response body for status WAL replay endpoint. + StatusSelfMetricsOutputBody: + type: object + properties: + status: + type: string + enum: + - success + - error + description: Response status. + example: success + data: + description: Response data (structure varies by endpoint). + example: + result: ok + warnings: + type: array + items: + type: string + description: Only set if there were warnings while executing the request. There will still be data in the data field. + infos: + type: array + items: + type: string + description: Only set if there were info-level annotations while executing the request. + required: + - status + - data + additionalProperties: false + description: Generic response body. DeleteSeriesOutputBody: type: object properties: diff --git a/web/ui/mantine-ui/src/api/responseTypes/selfMetrics.ts b/web/ui/mantine-ui/src/api/responseTypes/selfMetrics.ts new file mode 100644 index 0000000000..c47374ae3b --- /dev/null +++ b/web/ui/mantine-ui/src/api/responseTypes/selfMetrics.ts @@ -0,0 +1,91 @@ +// Result type for /api/v1/status/self_metrics endpoint. +// The response uses the standard ProtoJSON format for io.prometheus.client.MetricFamily. +// See https://protobuf.dev/programming-guides/json/ + +export interface ProtoLabelPair { + name: string; + value: string; +} + +export interface ProtoGauge { + value: number; +} + +export interface ProtoCounter { + value: number; + exemplar?: ProtoExemplar; + createdTimestamp?: string; +} + +export interface ProtoQuantile { + quantile: number; + value: number; +} + +export interface ProtoSummary { + sampleCount: string; + sampleSum: number; + quantile?: ProtoQuantile[]; + createdTimestamp?: string; +} + +export interface ProtoBucket { + cumulativeCount: string; + cumulativeCountFloat?: number; + upperBound: number; + exemplar?: ProtoExemplar; +} + +export interface ProtoBucketSpan { + offset: number; + length: number; +} + +export interface ProtoHistogram { + sampleCount: string; + sampleCountFloat?: number; + sampleSum: number; + bucket?: ProtoBucket[]; + createdTimestamp?: string; + schema?: number; + zeroThreshold?: number; + zeroCount?: string; + zeroCountFloat?: number; + negativeSpan?: ProtoBucketSpan[]; + negativeDelta?: string[]; + negativeCount?: number[]; + positiveSpan?: ProtoBucketSpan[]; + positiveDelta?: string[]; + positiveCount?: number[]; + exemplars?: ProtoExemplar[]; +} + +export interface ProtoExemplar { + label?: ProtoLabelPair[]; + value: number; + timestamp?: string; +} + +export interface ProtoUntyped { + value: number; +} + +export interface ProtoMetric { + label?: ProtoLabelPair[]; + gauge?: ProtoGauge; + counter?: ProtoCounter; + summary?: ProtoSummary; + histogram?: ProtoHistogram; + untyped?: ProtoUntyped; + timestampMs?: string; +} + +export interface ProtoMetricFamily { + name: string; + help: string; + type: string; + metric: ProtoMetric[]; + unit?: string; +} + +export type SelfMetricsResult = ProtoMetricFamily[];