diff --git a/.travis.yml b/.travis.yml index 3612d19dd7..78c382486f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,7 @@ sudo: false language: go go: -- 1.7.3 +- 1.7.4 go_import_path: github.com/prometheus/prometheus diff --git a/AUTHORS.md b/AUTHORS.md deleted file mode 100644 index dddaa986a8..0000000000 --- a/AUTHORS.md +++ /dev/null @@ -1,13 +0,0 @@ -The Prometheus project was started by Matt T. Proud (emeritus) and -Julius Volz in 2012. - -Maintainers of this repository: - -* Björn Rabenstein -* Fabian Reinartz -* Julius Volz - -More than [100 individuals][1] have contributed to this repository. Please -refer to the Git commit log for a complete list. - -[1]: https://github.com/prometheus/prometheus/graphs/contributors diff --git a/CHANGELOG.md b/CHANGELOG.md index 86b4370b37..9149f3a818 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,18 @@ +## 1.5.2 / 2017-02-10 + +* [BUGFIX] Fix series corruption in a special case of series maintenance where + the minimum series-file-shrink-ratio kicks in. +* [BUGFIX] Fix two panic conditions both related to processing a series + scheduled to be quarantined. +* [ENHANCEMENT] Binaries built with Go1.7.5. + +## 1.5.1 / 2017-02-07 + +* [BUGFIX] Don't lose fully persisted memory series during checkpointing. +* [BUGFIX] Fix intermittently failing relabeling. +* [BUGFIX] Make `-storage.local.series-file-shrink-ratio` work. +* [BUGFIX] Remove race condition from TestLoop. + ## 1.5.0 / 2017-01-23 * [CHANGE] Use lexicographic order to sort alerts by name. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5705f0fbea..dde3851b35 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,9 +2,9 @@ Prometheus uses GitHub to manage reviews of pull requests. -* If you have a trivial fix or improvement, go ahead and create a pull - request, addressing (with `@...`) one or more of the maintainers - (see [AUTHORS.md](AUTHORS.md)) in the description of the pull request. +* If you have a trivial fix or improvement, go ahead and create a pull request, + addressing (with `@...`) a suitable maintainer of this repository (see + [MAINTAINERS.md](MAINTAINERS.md)) in the description of the pull request. * If you plan to do something more involved, first discuss your ideas on our [mailing list](https://groups.google.com/forum/?fromgroups#!forum/prometheus-developers). diff --git a/MAINTAINERS.md b/MAINTAINERS.md new file mode 100644 index 0000000000..c55b0530ef --- /dev/null +++ b/MAINTAINERS.md @@ -0,0 +1,7 @@ +Maintainers of this repository with their focus areas: + +* Björn Rabenstein : Local storage; general code-level issues. +* Brian Brazil : Console templates; semantics of PromQL, service discovery, and relabeling. +* Fabian Reinartz : PromQL parsing and evaluation; implementation of retrieval, alert notification, and service discovery. +* Julius Volz : Remote storage integrations; web UI. + diff --git a/VERSION b/VERSION index bc80560fad..4cda8f19ed 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.5.0 +1.5.2 diff --git a/cmd/prometheus/config.go b/cmd/prometheus/config.go index 5e5f45c15b..0e75080c5d 100644 --- a/cmd/prometheus/config.go +++ b/cmd/prometheus/config.go @@ -53,7 +53,6 @@ var cfg = struct { alertmanagerURLs stringset prometheusURL string - influxdbURL string }{ alertmanagerURLs: stringset{}, } @@ -130,6 +129,10 @@ func init() { &cfg.tsdb.AppendableBlocks, "storage.tsdb.AppendableBlocks", 2, "Number of head blocks that can be appended to.", ) + cfg.fs.StringVar( + &cfg.localStorageEngine, "storage.local.engine", "persisted", + "Local storage engine. Supported values are: 'persisted' (full local storage with on-disk persistence) and 'none' (no local storage).", + ) // Alertmanager. cfg.fs.IntVar( diff --git a/cmd/prometheus/config_test.go b/cmd/prometheus/config_test.go index 5edb7179ca..711968a215 100644 --- a/cmd/prometheus/config_test.go +++ b/cmd/prometheus/config_test.go @@ -41,7 +41,6 @@ func TestParse(t *testing.T) { for i, test := range tests { // reset "immutable" config cfg.prometheusURL = "" - cfg.influxdbURL = "" cfg.alertmanagerURLs = stringset{} err := parse(test.input) @@ -52,46 +51,3 @@ func TestParse(t *testing.T) { } } } - -func TestParseAlertmanagerURLToConfig(t *testing.T) { - tests := []struct { - url string - username string - password string - }{ - { - url: "http://alertmanager.company.com", - username: "", - password: "", - }, - { - url: "https://user:password@alertmanager.company.com", - username: "user", - password: "password", - }, - } - - for i, test := range tests { - acfg, err := parseAlertmanagerURLToConfig(test.url) - if err != nil { - t.Errorf("%d. expected alertmanager URL to be valid, got %s", i, err) - } - - if acfg.HTTPClientConfig.BasicAuth != nil { - if test.username != acfg.HTTPClientConfig.BasicAuth.Username { - t.Errorf("%d. expected alertmanagerConfig username to be %q, got %q", - i, test.username, acfg.HTTPClientConfig.BasicAuth.Username) - } - - if test.password != acfg.HTTPClientConfig.BasicAuth.Password { - t.Errorf("%d. expected alertmanagerConfig password to be %q, got %q", i, - test.password, acfg.HTTPClientConfig.BasicAuth.Username) - } - continue - } - - if test.username != "" || test.password != "" { - t.Errorf("%d. expected alertmanagerConfig to have basicAuth filled, but was not", i) - } - } -} diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go index 33a34d0be9..39aecfdbff 100644 --- a/cmd/prometheus/main.go +++ b/cmd/prometheus/main.go @@ -91,9 +91,9 @@ func Main() int { return 1 } - // reloadableRemoteStorage := remote.New() - // sampleAppender = append(sampleAppender, reloadableRemoteStorage) - // reloadables = append(reloadables, reloadableRemoteStorage) + // remoteStorage := &remote.Storage{} + // sampleAppender = append(sampleAppender, remoteStorage) + // reloadables = append(reloadables, remoteStorage) var ( notifier = notifier.New(&cfg.notifier) @@ -172,12 +172,8 @@ func Main() int { } }() - // defer reloadableRemoteStorage.Stop() + // defer remoteStorage.Stop() - // The storage has to be fully initialized before registering. - if instrumentedStorage, ok := localStorage.(prometheus.Collector); ok { - prometheus.MustRegister(instrumentedStorage) - } prometheus.MustRegister(notifier) prometheus.MustRegister(configSuccess) prometheus.MustRegister(configSuccessTime) diff --git a/config/config.go b/config/config.go index ae00463299..93d4c67e41 100644 --- a/config/config.go +++ b/config/config.go @@ -204,7 +204,7 @@ type Config struct { RuleFiles []string `yaml:"rule_files,omitempty"` ScrapeConfigs []*ScrapeConfig `yaml:"scrape_configs,omitempty"` - RemoteWriteConfig RemoteWriteConfig `yaml:"remote_write,omitempty"` + RemoteWriteConfigs []*RemoteWriteConfig `yaml:"remote_write,omitempty"` // Catches all undefined fields and must be empty after parsing. XXX map[string]interface{} `yaml:",inline"` diff --git a/config/config_test.go b/config/config_test.go index a8c1f32e74..4be6989b7b 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -26,6 +26,14 @@ import ( "gopkg.in/yaml.v2" ) +func mustParseURL(u string) *URL { + parsed, err := url.Parse(u) + if err != nil { + panic(err) + } + return &URL{URL: parsed} +} + var expectedConf = &Config{ GlobalConfig: GlobalConfig{ ScrapeInterval: model.Duration(15 * time.Second), @@ -44,17 +52,24 @@ var expectedConf = &Config{ "testdata/my/*.rules", }, - RemoteWriteConfig: RemoteWriteConfig{ - RemoteTimeout: model.Duration(30 * time.Second), - WriteRelabelConfigs: []*RelabelConfig{ - { - SourceLabels: model.LabelNames{"__name__"}, - Separator: ";", - Regex: MustNewRegexp("expensive.*"), - Replacement: "$1", - Action: RelabelDrop, + RemoteWriteConfigs: []*RemoteWriteConfig{ + { + URL: mustParseURL("http://remote1/push"), + RemoteTimeout: model.Duration(30 * time.Second), + WriteRelabelConfigs: []*RelabelConfig{ + { + SourceLabels: model.LabelNames{"__name__"}, + Separator: ";", + Regex: MustNewRegexp("expensive.*"), + Replacement: "$1", + Action: RelabelDrop, + }, }, }, + { + URL: mustParseURL("http://remote2/push"), + RemoteTimeout: model.Duration(30 * time.Second), + }, }, ScrapeConfigs: []*ScrapeConfig{ diff --git a/config/testdata/conf.good.yml b/config/testdata/conf.good.yml index a3aca858a4..7fc6161138 100644 --- a/config/testdata/conf.good.yml +++ b/config/testdata/conf.good.yml @@ -14,10 +14,12 @@ rule_files: - "my/*.rules" remote_write: - write_relabel_configs: - - source_labels: [__name__] - regex: expensive.* - action: drop + - url: http://remote1/push + write_relabel_configs: + - source_labels: [__name__] + regex: expensive.* + action: drop + - url: http://remote2/push scrape_configs: - job_name: prometheus diff --git a/discovery/dns/dns.go b/discovery/dns/dns.go index ca9ed56fc0..9890de4aee 100644 --- a/discovery/dns/dns.go +++ b/discovery/dns/dns.go @@ -179,13 +179,12 @@ func lookupAll(name string, qtype uint16) (*dns.Msg, error) { for _, server := range conf.Servers { servAddr := net.JoinHostPort(server, conf.Port) - for _, suffix := range conf.Search { - response, err = lookup(name, qtype, client, servAddr, suffix, false) + for _, lname := range conf.NameList(name) { + response, err = lookup(lname, qtype, client, servAddr, false) if err != nil { log. With("server", server). With("name", name). - With("suffix", suffix). With("reason", err). Warn("DNS resolution failed.") continue @@ -194,22 +193,12 @@ func lookupAll(name string, qtype uint16) (*dns.Msg, error) { return response, nil } } - response, err = lookup(name, qtype, client, servAddr, "", false) - if err == nil { - return response, nil - } - log. - With("server", server). - With("name", name). - With("reason", err). - Warn("DNS resolution failed.") } return response, fmt.Errorf("could not resolve %s: no server responded", name) } -func lookup(name string, queryType uint16, client *dns.Client, servAddr string, suffix string, edns bool) (*dns.Msg, error) { +func lookup(lname string, queryType uint16, client *dns.Client, servAddr string, edns bool) (*dns.Msg, error) { msg := &dns.Msg{} - lname := strings.Join([]string{name, suffix}, ".") msg.SetQuestion(dns.Fqdn(lname), queryType) if edns { @@ -224,7 +213,7 @@ func lookup(name string, queryType uint16, client *dns.Client, servAddr string, if edns { // Truncated even though EDNS is used client.Net = "tcp" } - return lookup(name, queryType, client, servAddr, suffix, !edns) + return lookup(lname, queryType, client, servAddr, !edns) } if err != nil { return nil, err diff --git a/documentation/examples/prometheus-kubernetes.yml b/documentation/examples/prometheus-kubernetes.yml index 6d5f7165eb..8ac7899b2e 100644 --- a/documentation/examples/prometheus-kubernetes.yml +++ b/documentation/examples/prometheus-kubernetes.yml @@ -108,7 +108,7 @@ scrape_configs: - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port] action: replace target_label: __address__ - regex: (.+)(?::\d+);(\d+) + regex: ([^:]+)(?::\d+)?;(\d+) replacement: $1:$2 - action: labelmap regex: __meta_kubernetes_service_label_(.+) @@ -146,7 +146,7 @@ scrape_configs: target_label: instance - action: labelmap regex: __meta_kubernetes_service_label_(.+) - - source_labels: [__meta_kubernetes_service_namespace] + - source_labels: [__meta_kubernetes_namespace] target_label: kubernetes_namespace - source_labels: [__meta_kubernetes_service_name] target_label: kubernetes_name @@ -174,8 +174,8 @@ scrape_configs: regex: (.+) - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] action: replace - regex: (.+):(?:\d+);(\d+) - replacement: ${1}:${2} + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 target_label: __address__ - action: labelmap regex: __meta_kubernetes_pod_label_(.+) diff --git a/documentation/examples/remote_storage/README.md b/documentation/examples/remote_storage/example_receiver/README.md similarity index 94% rename from documentation/examples/remote_storage/README.md rename to documentation/examples/remote_storage/example_receiver/README.md index 483bb22dce..3a0be8c0ba 100644 --- a/documentation/examples/remote_storage/README.md +++ b/documentation/examples/remote_storage/example_receiver/README.md @@ -7,7 +7,7 @@ To use it: ``` go build -./remote_storage +./example_receiver ``` ...and then add the following to your `prometheus.yml`: diff --git a/documentation/examples/remote_storage/server.go b/documentation/examples/remote_storage/example_receiver/server.go similarity index 100% rename from documentation/examples/remote_storage/server.go rename to documentation/examples/remote_storage/example_receiver/server.go diff --git a/documentation/examples/remote_storage/remote_storage_bridge/README.md b/documentation/examples/remote_storage/remote_storage_bridge/README.md new file mode 100644 index 0000000000..ad194c7169 --- /dev/null +++ b/documentation/examples/remote_storage/remote_storage_bridge/README.md @@ -0,0 +1,35 @@ +# Remote storage bridge + +This is a bridge that receives samples in Prometheus's remote storage +format and forwards them to Graphite, InfluxDB, or OpenTSDB. It is meant +as a replacement for the built-in specific remote storage implementations +that have been removed from Prometheus. + +## Building + +``` +go build +``` + +## Running + +Example: + +``` +./remote_storage_bridge -graphite-address=localhost:8080 -opentsdb-url=http://localhost:8081/ +``` + +To show all flags: + +``` +./remote_storage_bridge -h +``` + +## Configuring Prometheus + +To configure Prometheus to send samples to this bridge, add the following to your `prometheus.yml`: + +```yaml +remote_write: + url: "http://localhost:9201/receive" +``` \ No newline at end of file diff --git a/documentation/examples/remote_storage/remote_storage_bridge/graphite/client.go b/documentation/examples/remote_storage/remote_storage_bridge/graphite/client.go new file mode 100644 index 0000000000..8ce28c4a58 --- /dev/null +++ b/documentation/examples/remote_storage/remote_storage_bridge/graphite/client.go @@ -0,0 +1,107 @@ +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package graphite + +import ( + "bytes" + "fmt" + "math" + "net" + "sort" + "time" + + "github.com/prometheus/common/log" + "github.com/prometheus/common/model" +) + +// Client allows sending batches of Prometheus samples to Graphite. +type Client struct { + address string + transport string + timeout time.Duration + prefix string +} + +// NewClient creates a new Client. +func NewClient(address string, transport string, timeout time.Duration, prefix string) *Client { + return &Client{ + address: address, + transport: transport, + timeout: timeout, + prefix: prefix, + } +} + +func pathFromMetric(m model.Metric, prefix string) string { + var buffer bytes.Buffer + + buffer.WriteString(prefix) + buffer.WriteString(escape(m[model.MetricNameLabel])) + + // We want to sort the labels. + labels := make(model.LabelNames, 0, len(m)) + for l := range m { + labels = append(labels, l) + } + sort.Sort(labels) + + // For each label, in order, add ".