diff --git a/.golangci.yml b/.golangci.yml index 166b2e0d48..cc290b3343 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -60,6 +60,10 @@ linters-settings: desc: "Use github.com/grafana/regexp instead of regexp" - pkg: "github.com/pkg/errors" desc: "Use 'errors' or 'fmt' instead of github.com/pkg/errors" + - pkg: "gzip" + desc: "Use github.com/klauspost/compress instead of gzip" + - pkg: "zlib" + desc: "Use github.com/klauspost/compress instead of zlib" errcheck: exclude-functions: # Don't flag lines such as "io.Copy(io.Discard, resp.Body)". diff --git a/cmd/promtool/archive.go b/cmd/promtool/archive.go index 6edb741acd..7b565c57cc 100644 --- a/cmd/promtool/archive.go +++ b/cmd/promtool/archive.go @@ -15,9 +15,10 @@ package main import ( "archive/tar" - "compress/gzip" "fmt" "os" + + "github.com/klauspost/compress/gzip" ) const filePerm = 0o666 diff --git a/model/textparse/promparse_test.go b/model/textparse/promparse_test.go index d82bfe598d..cbfc8aa6cc 100644 --- a/model/textparse/promparse_test.go +++ b/model/textparse/promparse_test.go @@ -15,15 +15,16 @@ package textparse import ( "bytes" - "compress/gzip" "errors" "io" "os" "testing" + "github.com/klauspost/compress/gzip" + "github.com/stretchr/testify/require" + "github.com/prometheus/common/expfmt" "github.com/prometheus/common/model" - "github.com/stretchr/testify/require" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/util/testutil" diff --git a/scrape/scrape.go b/scrape/scrape.go index dfa945852c..de35b9e408 100644 --- a/scrape/scrape.go +++ b/scrape/scrape.go @@ -16,7 +16,6 @@ package scrape import ( "bufio" "bytes" - "compress/gzip" "context" "errors" "fmt" @@ -31,6 +30,7 @@ import ( "github.com/go-kit/log" "github.com/go-kit/log/level" + "github.com/klauspost/compress/gzip" config_util "github.com/prometheus/common/config" "github.com/prometheus/common/model" "github.com/prometheus/common/version" diff --git a/scrape/scrape_test.go b/scrape/scrape_test.go index badeee2d1c..eefa7be66f 100644 --- a/scrape/scrape_test.go +++ b/scrape/scrape_test.go @@ -25,6 +25,7 @@ import ( "net/http" "net/http/httptest" "net/url" + "strconv" "strings" "sync" "testing" @@ -3518,3 +3519,76 @@ func TestPickSchema(t *testing.T) { require.Equal(t, tc.schema, schema) } } + +func BenchmarkTargetScraperGzip(b *testing.B) { + scenarios := []struct { + metricsCount int + body []byte + }{ + {metricsCount: 1}, + {metricsCount: 100}, + {metricsCount: 1000}, + {metricsCount: 10000}, + {metricsCount: 100000}, + } + + for i := 0; i < len(scenarios); i++ { + var buf bytes.Buffer + var name string + gw := gzip.NewWriter(&buf) + for j := 0; j < scenarios[i].metricsCount; j++ { + name = fmt.Sprintf("go_memstats_alloc_bytes_total_%d", j) + fmt.Fprintf(gw, "# HELP %s Total number of bytes allocated, even if freed.\n", name) + fmt.Fprintf(gw, "# TYPE %s counter\n", name) + fmt.Fprintf(gw, "%s %d\n", name, i*j) + } + gw.Close() + scenarios[i].body = buf.Bytes() + } + + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", `text/plain; version=0.0.4`) + w.Header().Set("Content-Encoding", "gzip") + for _, scenario := range scenarios { + if strconv.Itoa(scenario.metricsCount) == r.URL.Query()["count"][0] { + w.Write(scenario.body) + return + } + } + w.WriteHeader(http.StatusBadRequest) + }) + + server := httptest.NewServer(handler) + defer server.Close() + + serverURL, err := url.Parse(server.URL) + if err != nil { + panic(err) + } + + client, err := config_util.NewClientFromConfig(config_util.DefaultHTTPClientConfig, "test_job") + if err != nil { + panic(err) + } + + for _, scenario := range scenarios { + b.Run(fmt.Sprintf("metrics=%d", scenario.metricsCount), func(b *testing.B) { + ts := &targetScraper{ + Target: &Target{ + labels: labels.FromStrings( + model.SchemeLabel, serverURL.Scheme, + model.AddressLabel, serverURL.Host, + ), + params: url.Values{"count": []string{strconv.Itoa(scenario.metricsCount)}}, + }, + client: client, + timeout: time.Second, + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err = ts.scrape(context.Background()) + require.NoError(b, err) + } + }) + } +} diff --git a/util/httputil/compression.go b/util/httputil/compression.go index b96c088cbb..9a8a666453 100644 --- a/util/httputil/compression.go +++ b/util/httputil/compression.go @@ -14,11 +14,12 @@ package httputil import ( - "compress/gzip" - "compress/zlib" "io" "net/http" "strings" + + "github.com/klauspost/compress/gzip" + "github.com/klauspost/compress/zlib" ) const ( diff --git a/util/httputil/compression_test.go b/util/httputil/compression_test.go index 8512797613..2db6810bd7 100644 --- a/util/httputil/compression_test.go +++ b/util/httputil/compression_test.go @@ -15,13 +15,13 @@ package httputil import ( "bytes" - "compress/gzip" - "compress/zlib" "io" "net/http" "net/http/httptest" "testing" + "github.com/klauspost/compress/gzip" + "github.com/klauspost/compress/zlib" "github.com/stretchr/testify/require" )