mirror of
https://github.com/prometheus/prometheus.git
synced 2025-08-05 21:57:09 +02:00
discovery: add STACKIT SD (#16401)
This commit is contained in:
parent
5a1cce4fbb
commit
ceaa3bd6f9
@ -11,6 +11,7 @@ Maintainers for specific parts of the codebase:
|
||||
* `discovery`
|
||||
* `azure`: Jan-Otto Kröpke (<mail@jkroepke.de> / @jkroepke)
|
||||
* `k8s`: Frederic Branczyk (<fbranczyk@gmail.com> / @brancz)
|
||||
* `stackit`: Jan-Otto Kröpke (<mail@jkroepke.de> / @jkroepke)
|
||||
* `documentation`
|
||||
* `prometheus-mixin`: Matthias Loibl (<mail@matthiasloibl.com> / @metalmatze)
|
||||
* `model/histogram` and other code related to native histograms: Björn Rabenstein (<beorn@grafana.com> / @beorn7),
|
||||
|
@ -50,6 +50,7 @@ import (
|
||||
"github.com/prometheus/prometheus/discovery/ovhcloud"
|
||||
"github.com/prometheus/prometheus/discovery/puppetdb"
|
||||
"github.com/prometheus/prometheus/discovery/scaleway"
|
||||
"github.com/prometheus/prometheus/discovery/stackit"
|
||||
"github.com/prometheus/prometheus/discovery/targetgroup"
|
||||
"github.com/prometheus/prometheus/discovery/triton"
|
||||
"github.com/prometheus/prometheus/discovery/uyuni"
|
||||
@ -1473,6 +1474,45 @@ var expectedConf = &Config{
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
JobName: "stackit-servers",
|
||||
HonorTimestamps: true,
|
||||
ScrapeInterval: model.Duration(15 * time.Second),
|
||||
ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout,
|
||||
EnableCompression: true,
|
||||
BodySizeLimit: globBodySizeLimit,
|
||||
SampleLimit: globSampleLimit,
|
||||
TargetLimit: globTargetLimit,
|
||||
LabelLimit: globLabelLimit,
|
||||
LabelNameLengthLimit: globLabelNameLengthLimit,
|
||||
LabelValueLengthLimit: globLabelValueLengthLimit,
|
||||
ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols,
|
||||
ScrapeFailureLogFile: globScrapeFailureLogFile,
|
||||
MetricNameValidationScheme: UTF8ValidationConfig,
|
||||
MetricNameEscapingScheme: model.AllowUTF8,
|
||||
AlwaysScrapeClassicHistograms: boolPtr(false),
|
||||
ConvertClassicHistogramsToNHCB: boolPtr(false),
|
||||
|
||||
MetricsPath: DefaultScrapeConfig.MetricsPath,
|
||||
Scheme: DefaultScrapeConfig.Scheme,
|
||||
HTTPClientConfig: config.DefaultHTTPClientConfig,
|
||||
ServiceDiscoveryConfigs: discovery.Configs{
|
||||
&stackit.SDConfig{
|
||||
Project: "11111111-1111-1111-1111-111111111111",
|
||||
Region: "eu01",
|
||||
HTTPClientConfig: config.HTTPClientConfig{
|
||||
Authorization: &config.Authorization{
|
||||
Type: "Bearer",
|
||||
Credentials: "abcdef",
|
||||
},
|
||||
FollowRedirects: true,
|
||||
EnableHTTP2: true,
|
||||
},
|
||||
Port: 80,
|
||||
RefreshInterval: model.Duration(60 * time.Second),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
JobName: "uyuni",
|
||||
|
||||
@ -1922,7 +1962,7 @@ func TestElideSecrets(t *testing.T) {
|
||||
yamlConfig := string(config)
|
||||
|
||||
matches := secretRe.FindAllStringIndex(yamlConfig, -1)
|
||||
require.Len(t, matches, 24, "wrong number of secret matches found")
|
||||
require.Len(t, matches, 25, "wrong number of secret matches found")
|
||||
require.NotContains(t, yamlConfig, "mysecret",
|
||||
"yaml marshal reveals authentication credentials.")
|
||||
}
|
||||
@ -2429,6 +2469,10 @@ var expectedErrors = []struct {
|
||||
filename: "scrape_config_utf8_conflicting.bad.yml",
|
||||
errMsg: `utf8 metric names requested but validation scheme is not set to UTF8`,
|
||||
},
|
||||
{
|
||||
filename: "stackit_endpoint.bad.yml",
|
||||
errMsg: "invalid endpoint",
|
||||
},
|
||||
}
|
||||
|
||||
func TestBadConfigs(t *testing.T) {
|
||||
|
6
config/testdata/conf.good.yml
vendored
6
config/testdata/conf.good.yml
vendored
@ -417,6 +417,12 @@ scrape_configs:
|
||||
- authorization:
|
||||
credentials: abcdef
|
||||
|
||||
- job_name: stackit-servers
|
||||
stackit_sd_configs:
|
||||
- project: 11111111-1111-1111-1111-111111111111
|
||||
authorization:
|
||||
credentials: abcdef
|
||||
|
||||
- job_name: uyuni
|
||||
uyuni_sd_configs:
|
||||
- server: https://localhost:1234
|
||||
|
4
config/testdata/stackit_endpoint.bad.yml
vendored
Normal file
4
config/testdata/stackit_endpoint.bad.yml
vendored
Normal file
@ -0,0 +1,4 @@
|
||||
scrape_configs:
|
||||
- job_name: stackit
|
||||
stackit_sd_configs:
|
||||
- endpoint: "://invalid"
|
@ -36,6 +36,7 @@ import (
|
||||
_ "github.com/prometheus/prometheus/discovery/ovhcloud" // register ovhcloud
|
||||
_ "github.com/prometheus/prometheus/discovery/puppetdb" // register puppetdb
|
||||
_ "github.com/prometheus/prometheus/discovery/scaleway" // register scaleway
|
||||
_ "github.com/prometheus/prometheus/discovery/stackit" // register stackit
|
||||
_ "github.com/prometheus/prometheus/discovery/triton" // register triton
|
||||
_ "github.com/prometheus/prometheus/discovery/uyuni" // register uyuni
|
||||
_ "github.com/prometheus/prometheus/discovery/vultr" // register vultr
|
||||
|
32
discovery/stackit/metrics.go
Normal file
32
discovery/stackit/metrics.go
Normal file
@ -0,0 +1,32 @@
|
||||
// Copyright 2015 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package stackit
|
||||
|
||||
import (
|
||||
"github.com/prometheus/prometheus/discovery"
|
||||
)
|
||||
|
||||
var _ discovery.DiscovererMetrics = (*stackitMetrics)(nil)
|
||||
|
||||
type stackitMetrics struct {
|
||||
refreshMetrics discovery.RefreshMetricsInstantiator
|
||||
}
|
||||
|
||||
// Register implements discovery.DiscovererMetrics.
|
||||
func (m *stackitMetrics) Register() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Unregister implements discovery.DiscovererMetrics.
|
||||
func (m *stackitMetrics) Unregister() {}
|
162
discovery/stackit/mock_test.go
Normal file
162
discovery/stackit/mock_test.go
Normal file
@ -0,0 +1,162 @@
|
||||
// Copyright 2020 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package stackit
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// SDMock is the interface for the STACKIT IAAS API mock.
|
||||
type SDMock struct {
|
||||
t *testing.T
|
||||
Server *httptest.Server
|
||||
Mux *http.ServeMux
|
||||
}
|
||||
|
||||
// NewSDMock returns a new SDMock.
|
||||
func NewSDMock(t *testing.T) *SDMock {
|
||||
return &SDMock{
|
||||
t: t,
|
||||
}
|
||||
}
|
||||
|
||||
// Endpoint returns the URI to the mock server.
|
||||
func (m *SDMock) Endpoint() string {
|
||||
return m.Server.URL + "/"
|
||||
}
|
||||
|
||||
// Setup creates the mock server.
|
||||
func (m *SDMock) Setup() {
|
||||
m.Mux = http.NewServeMux()
|
||||
m.Server = httptest.NewServer(m.Mux)
|
||||
m.t.Cleanup(m.Server.Close)
|
||||
}
|
||||
|
||||
// ShutdownServer creates the mock server.
|
||||
func (m *SDMock) ShutdownServer() {
|
||||
m.Server.Close()
|
||||
}
|
||||
|
||||
const (
|
||||
testToken = "LRK9DAWQ1ZAEFSrCNEEzLCUwhYX1U3g7wMg4dTlkkDC96fyDuyJ39nVbVjCKSDfj"
|
||||
testProjectID = "00000000-0000-0000-0000-000000000000"
|
||||
)
|
||||
|
||||
// HandleServers mocks the STACKIT IAAS API.
|
||||
func (m *SDMock) HandleServers() {
|
||||
// /token endpoint mocks the token endpoint for service account authentication.
|
||||
// It checks if the request body starts with "assertion=ey" to simulate a valid assertion
|
||||
// as defined in RFC 7523.
|
||||
m.Mux.HandleFunc("/token", func(w http.ResponseWriter, r *http.Request) {
|
||||
reqBody, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
_, _ = fmt.Fprint(w, err)
|
||||
return
|
||||
}
|
||||
|
||||
// Expecting HTTP form encoded body with the field assertion.
|
||||
// JWT always start with "ey" (base64url encoded).
|
||||
if !bytes.HasPrefix(reqBody, []byte("assertion=ey")) {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Add("content-type", "application/json; charset=utf-8")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
|
||||
_, _ = fmt.Fprintf(w, `{"access_token": "%s"}`, testToken)
|
||||
})
|
||||
|
||||
m.Mux.HandleFunc(fmt.Sprintf("/v1/projects/%s/servers", testProjectID), func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Authorization") != fmt.Sprintf("Bearer %s", testToken) {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Add("content-type", "application/json; charset=utf-8")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
|
||||
_, _ = fmt.Fprint(w, `
|
||||
{
|
||||
"items": [
|
||||
{
|
||||
"availabilityZone": "eu01-3",
|
||||
"bootVolume": {
|
||||
"deleteOnTermination": false,
|
||||
"id": "1c15e4cc-8474-46be-b875-b473ea9fe80c"
|
||||
},
|
||||
"createdAt": "2025-03-12T14:48:17Z",
|
||||
"id": "b4176700-596a-4f80-9fc8-5f9c58a606e1",
|
||||
"labels": {
|
||||
"provisionSTACKITServerAgent": "true",
|
||||
"stackit_project_id": "00000000-0000-0000-0000-000000000000"
|
||||
},
|
||||
"launchedAt": "2025-03-12T14:48:52Z",
|
||||
"machineType": "g1.1",
|
||||
"name": "runcommandtest",
|
||||
"nics": [
|
||||
{
|
||||
"ipv4": "10.0.0.153",
|
||||
"mac": "fa:16:4f:42:1c:d3",
|
||||
"networkId": "3173494f-2f6c-490d-8c12-4b3c86b4338b",
|
||||
"networkName": "test",
|
||||
"publicIp": "192.0.2.1",
|
||||
"nicId": "b36097c5-e1c5-4e12-ae97-c03e144db127",
|
||||
"nicSecurity": true,
|
||||
"securityGroups": [
|
||||
"6e60809f-bed3-46c6-a39c-adddd6455674"
|
||||
]
|
||||
}
|
||||
],
|
||||
"powerStatus": "STOPPED",
|
||||
"serviceAccountMails": [],
|
||||
"status": "INACTIVE",
|
||||
"updatedAt": "2025-03-13T07:08:29Z",
|
||||
"userData": null,
|
||||
"volumes": [
|
||||
"1c15e4cc-8474-46be-b875-b473ea9fe80c"
|
||||
]
|
||||
},
|
||||
{
|
||||
"availabilityZone": "eu01-m",
|
||||
"bootVolume": {
|
||||
"deleteOnTermination": false,
|
||||
"id": "1e3ffe2b-878f-46e5-b39e-372e13a09551"
|
||||
},
|
||||
"createdAt": "2025-04-10T16:45:25Z",
|
||||
"id": "ee337436-1f15-4647-a03e-154009966179",
|
||||
"labels": {},
|
||||
"launchedAt": "2025-04-10T16:46:00Z",
|
||||
"machineType": "t1.1",
|
||||
"name": "server1",
|
||||
"nics": [],
|
||||
"powerStatus": "RUNNING",
|
||||
"serviceAccountMails": [],
|
||||
"status": "ACTIVE",
|
||||
"updatedAt": "2025-04-10T16:46:00Z",
|
||||
"volumes": [
|
||||
"1e3ffe2b-878f-46e5-b39e-372e13a09551"
|
||||
]
|
||||
}
|
||||
]
|
||||
}`,
|
||||
)
|
||||
})
|
||||
}
|
222
discovery/stackit/server.go
Normal file
222
discovery/stackit/server.go
Normal file
@ -0,0 +1,222 @@
|
||||
// Copyright 2020 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package stackit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/common/config"
|
||||
"github.com/prometheus/common/model"
|
||||
"github.com/stackitcloud/stackit-sdk-go/core/auth"
|
||||
stackitconfig "github.com/stackitcloud/stackit-sdk-go/core/config"
|
||||
|
||||
"github.com/prometheus/prometheus/discovery/refresh"
|
||||
"github.com/prometheus/prometheus/discovery/targetgroup"
|
||||
"github.com/prometheus/prometheus/util/strutil"
|
||||
)
|
||||
|
||||
const (
|
||||
stackitAPIEndpoint = "https://iaas.api.%s.stackit.cloud"
|
||||
|
||||
stackitLabelPrivateIPv4 = stackitLabelPrefix + "private_ipv4_"
|
||||
stackitLabelType = stackitLabelPrefix + "type"
|
||||
stackitLabelLabel = stackitLabelPrefix + "label_"
|
||||
stackitLabelLabelPresent = stackitLabelPrefix + "labelpresent_"
|
||||
)
|
||||
|
||||
// Discovery periodically performs STACKIT Cloud requests.
|
||||
// It implements the Discoverer interface.
|
||||
type iaasDiscovery struct {
|
||||
*refresh.Discovery
|
||||
httpClient *http.Client
|
||||
logger *slog.Logger
|
||||
apiEndpoint string
|
||||
project string
|
||||
port int
|
||||
}
|
||||
|
||||
// newServerDiscovery returns a new iaasDiscovery, which periodically refreshes its targets.
|
||||
func newServerDiscovery(conf *SDConfig, logger *slog.Logger) (*iaasDiscovery, error) {
|
||||
d := &iaasDiscovery{
|
||||
project: conf.Project,
|
||||
port: conf.Port,
|
||||
apiEndpoint: conf.Endpoint,
|
||||
logger: logger,
|
||||
}
|
||||
|
||||
rt, err := config.NewRoundTripperFromConfig(conf.HTTPClientConfig, "stackit_sd")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
d.apiEndpoint = conf.Endpoint
|
||||
if d.apiEndpoint == "" {
|
||||
d.apiEndpoint = fmt.Sprintf(stackitAPIEndpoint, conf.Region)
|
||||
}
|
||||
|
||||
servers := stackitconfig.ServerConfigurations{stackitconfig.ServerConfiguration{
|
||||
URL: d.apiEndpoint,
|
||||
Description: "STACKIT IAAS API",
|
||||
}}
|
||||
|
||||
d.httpClient = &http.Client{
|
||||
Timeout: time.Duration(conf.RefreshInterval),
|
||||
Transport: rt,
|
||||
}
|
||||
|
||||
stackitConfiguration := &stackitconfig.Configuration{
|
||||
UserAgent: userAgent,
|
||||
HTTPClient: d.httpClient,
|
||||
Servers: servers,
|
||||
NoAuth: conf.ServiceAccountKey == "" && conf.ServiceAccountKeyPath == "",
|
||||
|
||||
ServiceAccountKey: conf.ServiceAccountKey,
|
||||
PrivateKey: conf.PrivateKey,
|
||||
ServiceAccountKeyPath: conf.ServiceAccountKeyPath,
|
||||
PrivateKeyPath: conf.PrivateKeyPath,
|
||||
CredentialsFilePath: conf.CredentialsFilePath,
|
||||
}
|
||||
|
||||
if conf.tokenURL != "" {
|
||||
stackitConfiguration.TokenCustomUrl = conf.tokenURL
|
||||
}
|
||||
|
||||
authRoundTripper, err := auth.SetupAuth(stackitConfiguration)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("setting up authentication: %w", err)
|
||||
}
|
||||
|
||||
d.httpClient.Transport = authRoundTripper
|
||||
|
||||
return d, nil
|
||||
}
|
||||
|
||||
func (i *iaasDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
|
||||
apiURL, err := url.Parse(i.apiEndpoint)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid API endpoint URL %s: %w", i.apiEndpoint, err)
|
||||
}
|
||||
|
||||
apiURL.Path, err = url.JoinPath(apiURL.Path, "v1", "projects", i.project, "servers")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("joining URL path: %w", err)
|
||||
}
|
||||
|
||||
q := apiURL.Query()
|
||||
q.Set("details", "true")
|
||||
apiURL.RawQuery = q.Encode()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL.String(), nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating request: %w", err)
|
||||
}
|
||||
|
||||
req.Header.Set("Accept", "application/json")
|
||||
|
||||
res, err := i.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("sending request: %w", err)
|
||||
}
|
||||
|
||||
defer res.Body.Close()
|
||||
|
||||
if res.StatusCode != http.StatusOK {
|
||||
errorMessage, _ := io.ReadAll(res.Body)
|
||||
|
||||
return nil, fmt.Errorf("unexpected status code %d: %s", res.StatusCode, string(errorMessage))
|
||||
}
|
||||
|
||||
var serversResponse *ServerListResponse
|
||||
|
||||
if err := json.NewDecoder(res.Body).Decode(&serversResponse); err != nil {
|
||||
return nil, fmt.Errorf("decoding response: %w", err)
|
||||
}
|
||||
|
||||
if serversResponse == nil || serversResponse.Items == nil || len(*serversResponse.Items) == 0 {
|
||||
return []*targetgroup.Group{{Source: "stackit", Targets: []model.LabelSet{}}}, nil
|
||||
}
|
||||
|
||||
targets := make([]model.LabelSet, 0, len(*serversResponse.Items))
|
||||
for _, server := range *serversResponse.Items {
|
||||
if server.Nics == nil {
|
||||
i.logger.Debug("server has no network interfaces. Skipping", slog.String("server_id", server.ID))
|
||||
continue
|
||||
}
|
||||
|
||||
labels := model.LabelSet{
|
||||
stackitLabelProject: model.LabelValue(i.project),
|
||||
stackitLabelID: model.LabelValue(server.ID),
|
||||
stackitLabelName: model.LabelValue(server.Name),
|
||||
stackitLabelAvailabilityZone: model.LabelValue(server.AvailabilityZone),
|
||||
stackitLabelStatus: model.LabelValue(server.Status),
|
||||
stackitLabelPowerStatus: model.LabelValue(server.PowerStatus),
|
||||
stackitLabelType: model.LabelValue(server.MachineType),
|
||||
}
|
||||
|
||||
var (
|
||||
addressLabel string
|
||||
serverPublicIP string
|
||||
)
|
||||
|
||||
for _, nic := range server.Nics {
|
||||
if nic.PublicIP != nil && *nic.PublicIP != "" && serverPublicIP == "" {
|
||||
serverPublicIP = *nic.PublicIP
|
||||
addressLabel = serverPublicIP
|
||||
}
|
||||
|
||||
if nic.IPv4 != nil && *nic.IPv4 != "" {
|
||||
networkLabel := model.LabelName(stackitLabelPrivateIPv4 + strutil.SanitizeLabelName(nic.NetworkName))
|
||||
labels[networkLabel] = model.LabelValue(*nic.IPv4)
|
||||
if addressLabel == "" {
|
||||
addressLabel = *nic.IPv4
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if addressLabel == "" {
|
||||
// Skip servers without IPs.
|
||||
continue
|
||||
}
|
||||
|
||||
// Public IPs for servers are optional.
|
||||
if serverPublicIP != "" {
|
||||
labels[stackitLabelPublicIPv4] = model.LabelValue(serverPublicIP)
|
||||
}
|
||||
|
||||
labels[model.AddressLabel] = model.LabelValue(net.JoinHostPort(addressLabel, strconv.FormatUint(uint64(i.port), 10)))
|
||||
|
||||
for labelKey, labelValue := range server.Labels {
|
||||
if labelStringValue, ok := labelValue.(string); ok {
|
||||
presentLabel := model.LabelName(stackitLabelLabelPresent + strutil.SanitizeLabelName(labelKey))
|
||||
labels[presentLabel] = "true"
|
||||
|
||||
label := model.LabelName(stackitLabelLabel + strutil.SanitizeLabelName(labelKey))
|
||||
labels[label] = model.LabelValue(labelStringValue)
|
||||
}
|
||||
}
|
||||
|
||||
targets = append(targets, labels)
|
||||
}
|
||||
|
||||
return []*targetgroup.Group{{Source: "stackit", Targets: targets}}, nil
|
||||
}
|
131
discovery/stackit/server_test.go
Normal file
131
discovery/stackit/server_test.go
Normal file
@ -0,0 +1,131 @@
|
||||
// Copyright 2020 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package stackit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/x509"
|
||||
"encoding/pem"
|
||||
"testing"
|
||||
|
||||
"github.com/prometheus/common/model"
|
||||
"github.com/prometheus/common/promslog"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type serverSDTestSuite struct {
|
||||
Mock *SDMock
|
||||
}
|
||||
|
||||
func (s *serverSDTestSuite) SetupTest(t *testing.T) {
|
||||
s.Mock = NewSDMock(t)
|
||||
s.Mock.Setup()
|
||||
|
||||
s.Mock.HandleServers()
|
||||
}
|
||||
|
||||
func TestServerSDRefresh(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
cfg SDConfig
|
||||
}{
|
||||
{
|
||||
name: "default with token",
|
||||
cfg: func() SDConfig {
|
||||
cfg := DefaultSDConfig
|
||||
cfg.HTTPClientConfig.BearerToken = testToken
|
||||
|
||||
return cfg
|
||||
}(),
|
||||
},
|
||||
{
|
||||
name: "default with service account key",
|
||||
cfg: func() SDConfig {
|
||||
// Generate a new RSA key pair with a size of 2048 bits
|
||||
key, err := rsa.GenerateKey(rand.Reader, 2048)
|
||||
require.NoError(t, err)
|
||||
|
||||
cfg := DefaultSDConfig
|
||||
cfg.PrivateKey = string(pem.EncodeToMemory(&pem.Block{
|
||||
Type: "RSA PRIVATE KEY",
|
||||
Bytes: x509.MarshalPKCS1PrivateKey(key),
|
||||
}))
|
||||
|
||||
cfg.ServiceAccountKey = `{
|
||||
"Active": true,
|
||||
"CreatedAt": "2025-04-05T12:34:56Z",
|
||||
"Credentials": {
|
||||
"Aud": "https://stackit-service-account-prod.apps.01.cf.eu01.stackit.cloud",
|
||||
"Iss": "stackit@sa.stackit.cloud",
|
||||
"Kid": "123e4567-e89b-12d3-a456-426614174000",
|
||||
"Sub": "123e4567-e89b-12d3-a456-426614174001"
|
||||
},
|
||||
"ID": "123e4567-e89b-12d3-a456-426614174002",
|
||||
"KeyAlgorithm": "RSA_2048",
|
||||
"KeyOrigin": "USER_PROVIDED",
|
||||
"KeyType": "USER_MANAGED",
|
||||
"PublicKey": "...",
|
||||
"ValidUntil": "2025-04-05T13:34:56Z"
|
||||
}`
|
||||
|
||||
return cfg
|
||||
}(),
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
suite := &serverSDTestSuite{}
|
||||
suite.SetupTest(t)
|
||||
defer suite.Mock.ShutdownServer()
|
||||
|
||||
tc.cfg.Endpoint = suite.Mock.Endpoint()
|
||||
tc.cfg.tokenURL = suite.Mock.Endpoint() + "token"
|
||||
tc.cfg.Project = testProjectID
|
||||
|
||||
d, err := newServerDiscovery(&tc.cfg, promslog.NewNopLogger())
|
||||
require.NoError(t, err)
|
||||
|
||||
targetGroups, err := d.refresh(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.Len(t, targetGroups, 1)
|
||||
|
||||
targetGroup := targetGroups[0]
|
||||
require.NotNil(t, targetGroup, "targetGroup should not be nil")
|
||||
require.NotNil(t, targetGroup.Targets, "targetGroup.targets should not be nil")
|
||||
require.Len(t, targetGroup.Targets, 1)
|
||||
|
||||
for i, labelSet := range []model.LabelSet{
|
||||
{
|
||||
"__address__": model.LabelValue("192.0.2.1:80"),
|
||||
"__meta_stackit_project": model.LabelValue("00000000-0000-0000-0000-000000000000"),
|
||||
"__meta_stackit_id": model.LabelValue("b4176700-596a-4f80-9fc8-5f9c58a606e1"),
|
||||
"__meta_stackit_type": model.LabelValue("g1.1"),
|
||||
"__meta_stackit_private_ipv4_test": model.LabelValue("10.0.0.153"),
|
||||
"__meta_stackit_public_ipv4": model.LabelValue("192.0.2.1"),
|
||||
"__meta_stackit_labelpresent_provisionSTACKITServerAgent": model.LabelValue("true"),
|
||||
"__meta_stackit_label_provisionSTACKITServerAgent": model.LabelValue("true"),
|
||||
"__meta_stackit_labelpresent_stackit_project_id": model.LabelValue("true"),
|
||||
"__meta_stackit_name": model.LabelValue("runcommandtest"),
|
||||
"__meta_stackit_availability_zone": model.LabelValue("eu01-3"),
|
||||
"__meta_stackit_status": model.LabelValue("INACTIVE"),
|
||||
"__meta_stackit_power_status": model.LabelValue("STOPPED"),
|
||||
"__meta_stackit_label_stackit_project_id": model.LabelValue("00000000-0000-0000-0000-000000000000"),
|
||||
},
|
||||
} {
|
||||
require.Equal(t, labelSet, targetGroup.Targets[i])
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
153
discovery/stackit/stackit.go
Normal file
153
discovery/stackit/stackit.go
Normal file
@ -0,0 +1,153 @@
|
||||
// Copyright 2020 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package stackit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/common/config"
|
||||
"github.com/prometheus/common/model"
|
||||
"github.com/prometheus/common/version"
|
||||
|
||||
"github.com/prometheus/prometheus/discovery"
|
||||
"github.com/prometheus/prometheus/discovery/refresh"
|
||||
"github.com/prometheus/prometheus/discovery/targetgroup"
|
||||
)
|
||||
|
||||
const (
|
||||
stackitLabelPrefix = model.MetaLabelPrefix + "stackit_"
|
||||
stackitLabelProject = stackitLabelPrefix + "project"
|
||||
stackitLabelID = stackitLabelPrefix + "id"
|
||||
stackitLabelName = stackitLabelPrefix + "name"
|
||||
stackitLabelStatus = stackitLabelPrefix + "status"
|
||||
stackitLabelPowerStatus = stackitLabelPrefix + "power_status"
|
||||
stackitLabelAvailabilityZone = stackitLabelPrefix + "availability_zone"
|
||||
stackitLabelPublicIPv4 = stackitLabelPrefix + "public_ipv4"
|
||||
)
|
||||
|
||||
var userAgent = version.PrometheusUserAgent()
|
||||
|
||||
// DefaultSDConfig is the default STACKIT SD configuration.
|
||||
var DefaultSDConfig = SDConfig{
|
||||
Region: "eu01",
|
||||
Port: 80,
|
||||
RefreshInterval: model.Duration(60 * time.Second),
|
||||
HTTPClientConfig: config.DefaultHTTPClientConfig,
|
||||
}
|
||||
|
||||
func init() {
|
||||
discovery.RegisterConfig(&SDConfig{})
|
||||
}
|
||||
|
||||
// SDConfig is the configuration for STACKIT based service discovery.
|
||||
type SDConfig struct {
|
||||
HTTPClientConfig config.HTTPClientConfig `yaml:",inline"`
|
||||
|
||||
Project string `yaml:"project"`
|
||||
RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"`
|
||||
Port int `yaml:"port,omitempty"`
|
||||
Region string `yaml:"region,omitempty"`
|
||||
Endpoint string `yaml:"endpoint,omitempty"`
|
||||
ServiceAccountKey string `yaml:"service_account_key,omitempty"`
|
||||
PrivateKey string `yaml:"private_key,omitempty"`
|
||||
ServiceAccountKeyPath string `yaml:"service_account_key_path,omitempty"`
|
||||
PrivateKeyPath string `yaml:"private_key_path,omitempty"`
|
||||
CredentialsFilePath string `yaml:"credentials_file_path,omitempty"`
|
||||
|
||||
// For testing only
|
||||
tokenURL string
|
||||
}
|
||||
|
||||
// NewDiscovererMetrics implements discovery.Config.
|
||||
func (*SDConfig) NewDiscovererMetrics(_ prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics {
|
||||
return &stackitMetrics{
|
||||
refreshMetrics: rmi,
|
||||
}
|
||||
}
|
||||
|
||||
// Name returns the name of the Config.
|
||||
func (*SDConfig) Name() string { return "stackit" }
|
||||
|
||||
// NewDiscoverer returns a Discoverer for the Config.
|
||||
func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
|
||||
return NewDiscovery(c, opts.Logger, opts.Metrics)
|
||||
}
|
||||
|
||||
type refresher interface {
|
||||
refresh(context.Context) ([]*targetgroup.Group, error)
|
||||
}
|
||||
|
||||
// UnmarshalYAML implements the yaml.Unmarshaler interface.
|
||||
func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
||||
*c = DefaultSDConfig
|
||||
type plain SDConfig
|
||||
err := unmarshal((*plain)(c))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if c.Endpoint == "" && c.Region == "" {
|
||||
return errors.New("stackit_sd: endpoint and region missing")
|
||||
}
|
||||
|
||||
if _, err = url.Parse(c.Endpoint); err != nil {
|
||||
return fmt.Errorf("stackit_sd: invalid endpoint %q: %w", c.Endpoint, err)
|
||||
}
|
||||
|
||||
return c.HTTPClientConfig.Validate()
|
||||
}
|
||||
|
||||
// SetDirectory joins any relative file paths with dir.
|
||||
func (c *SDConfig) SetDirectory(dir string) {
|
||||
c.HTTPClientConfig.SetDirectory(dir)
|
||||
}
|
||||
|
||||
// Discovery periodically performs STACKIT API requests. It implements
|
||||
// the Discoverer interface.
|
||||
type Discovery struct {
|
||||
*refresh.Discovery
|
||||
}
|
||||
|
||||
// NewDiscovery returns a new Discovery which periodically refreshes its targets.
|
||||
func NewDiscovery(conf *SDConfig, logger *slog.Logger, metrics discovery.DiscovererMetrics) (*refresh.Discovery, error) {
|
||||
m, ok := metrics.(*stackitMetrics)
|
||||
if !ok {
|
||||
return nil, errors.New("invalid discovery metrics type")
|
||||
}
|
||||
|
||||
r, err := newRefresher(conf, logger)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return refresh.NewDiscovery(
|
||||
refresh.Options{
|
||||
Logger: logger,
|
||||
Mech: "stackit",
|
||||
Interval: time.Duration(conf.RefreshInterval),
|
||||
RefreshF: r.refresh,
|
||||
MetricsInstantiator: m.refreshMetrics,
|
||||
},
|
||||
), nil
|
||||
}
|
||||
|
||||
func newRefresher(conf *SDConfig, l *slog.Logger) (refresher, error) {
|
||||
return newServerDiscovery(conf, l)
|
||||
}
|
38
discovery/stackit/types.go
Normal file
38
discovery/stackit/types.go
Normal file
@ -0,0 +1,38 @@
|
||||
// Copyright 2020 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package stackit
|
||||
|
||||
// ServerListResponse Response object for server list request.
|
||||
// https://docs.api.eu01.stackit.cloud/documentation/iaas/version/v1#tag/Servers/operation/v1ListServersInProject
|
||||
type ServerListResponse struct {
|
||||
Items *[]Server `json:"items"`
|
||||
}
|
||||
|
||||
type Server struct {
|
||||
AvailabilityZone string `json:"availabilityZone"`
|
||||
ID string `json:"id"`
|
||||
Labels map[string]interface{} `json:"labels"`
|
||||
MachineType string `json:"machineType"`
|
||||
Name string `json:"name"`
|
||||
Nics []ServerNetwork `json:"nics"`
|
||||
PowerStatus string `json:"powerStatus"`
|
||||
Status string `json:"status"`
|
||||
}
|
||||
|
||||
// ServerNetwork Describes the object that matches servers to its networks.
|
||||
type ServerNetwork struct {
|
||||
NetworkName string `json:"networkName"`
|
||||
IPv4 *string `json:"ipv4,omitempty"`
|
||||
PublicIP *string `json:"publicIp,omitempty"`
|
||||
}
|
@ -431,6 +431,10 @@ scaleway_sd_configs:
|
||||
serverset_sd_configs:
|
||||
[ - <serverset_sd_config> ... ]
|
||||
|
||||
# List of STACKIT service discovery configurations.
|
||||
stackit_sd_configs:
|
||||
[ - <stackit_sd_config> ... ]
|
||||
|
||||
# List of Triton service discovery configurations.
|
||||
triton_sd_configs:
|
||||
[ - <triton_sd_config> ... ]
|
||||
@ -2258,6 +2262,70 @@ paths:
|
||||
|
||||
Serverset data must be in the JSON format, the Thrift format is not currently supported.
|
||||
|
||||
### `<stackit_sd_config>`
|
||||
|
||||
[STACKIT](https://www.stackit.de/de/) SD configurations allow retrieving
|
||||
scrape targets from various APIs.
|
||||
|
||||
The following meta labels are available on targets during [relabeling](#relabel_config):
|
||||
|
||||
* `__meta_stackit_availability_zone`: The availability zone of the server.
|
||||
* `__meta_stackit_label_<labelname>`: Each server label, with unsupported characters replaced by underscores.</labelname>
|
||||
* `__meta_stackit_labelpresent_<labelname>`: "true" for each label of the server, with unsupported characters replaced by underscores.</labelname>
|
||||
* `__meta_stackit_private_ipv4_<networkname>`: the private ipv4 address of the server within a given network
|
||||
* `__meta_stackit_public_ipv4`: the public ipv4 address of the server
|
||||
* `__meta_stackit_id`: The ID of the target.
|
||||
* `__meta_stackit_type`: The type or brand of the target.
|
||||
* `__meta_stackit_name`: The server name.
|
||||
* `__meta_stackit_status`: The current status of the server.
|
||||
* `__meta_stackit_power_status`: The power status of the server.
|
||||
|
||||
See below for the configuration options for STACKIT discovery:
|
||||
|
||||
```yaml
|
||||
# The STACKIT project
|
||||
project: <string>
|
||||
|
||||
# STACKIT region to use. No automatic discovery of the region is done.
|
||||
[ region : <string> | default = "eu01" ]
|
||||
|
||||
# Custom API endpoint to be used. Format scheme://host:port
|
||||
[ endpoint : <string> ]
|
||||
|
||||
# The port to scrape metrics from.
|
||||
[ port: <int> | default = 80 ]
|
||||
|
||||
# Raw private key string used for authenticating a service account
|
||||
[ private_key: <string> ]
|
||||
|
||||
# Path to a file containing the raw private key string
|
||||
[ private_key_path: <string> ]
|
||||
|
||||
# Full JSON-formatted service account key used for authentication
|
||||
[ service_account_key: <string> ]
|
||||
|
||||
# Path to a file containing the JSON-formatted service account key
|
||||
[ service_account_key_path: <string> ]
|
||||
|
||||
# Path to a file containing STACKIT credentials.
|
||||
[ credentials_file_path: <string> ]
|
||||
|
||||
# The time after which the servers are refreshed.
|
||||
[ refresh_interval: <duration> | default = 60s ]
|
||||
|
||||
# HTTP client settings, including authentication methods (such as basic auth and
|
||||
# authorization), proxy configurations, TLS options, custom HTTP headers, etc.
|
||||
[ <http_config> ]
|
||||
```
|
||||
|
||||
A Service Account Token can be set through `http_config`.
|
||||
|
||||
```yaml
|
||||
stackit_sd_config:
|
||||
- authorization:
|
||||
credentials: <token>
|
||||
```
|
||||
|
||||
### `<triton_sd_config>`
|
||||
|
||||
[Triton](https://github.com/joyent/triton) SD configurations allow retrieving
|
||||
@ -2830,6 +2898,10 @@ scaleway_sd_configs:
|
||||
serverset_sd_configs:
|
||||
[ - <serverset_sd_config> ... ]
|
||||
|
||||
# List of STACKIT service discovery configurations.
|
||||
stackit_sd_configs:
|
||||
[ - <stackit_sd_config> ... ]
|
||||
|
||||
# List of Triton service discovery configurations.
|
||||
triton_sd_configs:
|
||||
[ - <triton_sd_config> ... ]
|
||||
|
34
documentation/examples/prometheus-stackit.yml
Normal file
34
documentation/examples/prometheus-stackit.yml
Normal file
@ -0,0 +1,34 @@
|
||||
# A example scrape configuration for running Prometheus with
|
||||
# STACKIT.
|
||||
|
||||
scrape_configs:
|
||||
# Make Prometheus scrape itself for metrics.
|
||||
- job_name: "prometheus"
|
||||
static_configs:
|
||||
- targets: ["localhost:9090"]
|
||||
|
||||
# Discover Node Exporter instances to scrape.
|
||||
- job_name: "node"
|
||||
|
||||
stackit_sd_configs:
|
||||
- project: 11111111-1111-1111-1111-111111111111
|
||||
authorization:
|
||||
credentials: "<replace with a STACKIT ServiceAccount Token>"
|
||||
relabel_configs:
|
||||
# Use the public IPv4 and port 9100 to scrape the target.
|
||||
- source_labels: [__meta_stackit_public_ipv4]
|
||||
target_label: __address__
|
||||
replacement: "$1:9100"
|
||||
|
||||
# Discover Node Exporter instances to scrape using a STACKIT Subnet called mynet.
|
||||
- job_name: "node_private"
|
||||
|
||||
stackit_sd_configs:
|
||||
- project: 11111111-1111-1111-1111-111111111111
|
||||
authorization:
|
||||
credentials: "<replace with a STACKIT ServiceAccount Token>"
|
||||
relabel_configs:
|
||||
# Use the private IPv4 within the STACKIT Subnet and port 9100 to scrape the target.
|
||||
- source_labels: [__meta_stackit_private_ipv4_mynet]
|
||||
target_label: __address__
|
||||
replacement: "$1:9100"
|
1
go.mod
1
go.mod
@ -56,6 +56,7 @@ require (
|
||||
github.com/prometheus/sigv4 v0.1.2
|
||||
github.com/scaleway/scaleway-sdk-go v1.0.0-beta.33
|
||||
github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c
|
||||
github.com/stackitcloud/stackit-sdk-go/core v0.16.2
|
||||
github.com/stretchr/testify v1.10.0
|
||||
github.com/vultr/govultr/v2 v2.17.2
|
||||
go.opentelemetry.io/collector/component v1.31.0
|
||||
|
2
go.sum
2
go.sum
@ -459,6 +459,8 @@ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVs
|
||||
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
||||
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
|
||||
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/stackitcloud/stackit-sdk-go/core v0.16.2 h1:F8A4P/LLlQSbz0S0+G3m8rb3BUOK6EcR/CKx5UQY5jQ=
|
||||
github.com/stackitcloud/stackit-sdk-go/core v0.16.2/go.mod h1:8KIw3czdNJ9sdil9QQimxjR6vHjeINFrRv0iZ67wfn0=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
|
@ -16,6 +16,7 @@
|
||||
- github.com/prometheus/prometheus/discovery/ovhcloud
|
||||
- github.com/prometheus/prometheus/discovery/puppetdb
|
||||
- github.com/prometheus/prometheus/discovery/scaleway
|
||||
- github.com/prometheus/prometheus/discovery/stackit
|
||||
- github.com/prometheus/prometheus/discovery/triton
|
||||
- github.com/prometheus/prometheus/discovery/uyuni
|
||||
- github.com/prometheus/prometheus/discovery/vultr
|
||||
|
@ -52,6 +52,8 @@ import (
|
||||
_ "github.com/prometheus/prometheus/discovery/puppetdb"
|
||||
// Register scaleway plugin.
|
||||
_ "github.com/prometheus/prometheus/discovery/scaleway"
|
||||
// Register stackit plugin.
|
||||
_ "github.com/prometheus/prometheus/discovery/stackit"
|
||||
// Register triton plugin.
|
||||
_ "github.com/prometheus/prometheus/discovery/triton"
|
||||
// Register uyuni plugin.
|
||||
|
Loading…
Reference in New Issue
Block a user