diff --git a/.github/stale.yml b/.github/stale.yml
deleted file mode 100644
index 66a72af533..0000000000
--- a/.github/stale.yml
+++ /dev/null
@@ -1,56 +0,0 @@
-# Configuration for probot-stale - https://github.com/probot/stale
-
-# Number of days of inactivity before an Issue or Pull Request becomes stale
-daysUntilStale: 60
-
-# Number of days of inactivity before an Issue or Pull Request with the stale label is closed.
-# Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale.
-daysUntilClose: false
-
-# Only issues or pull requests with all of these labels are check if stale. Defaults to `[]` (disabled)
-onlyLabels: []
-
-# Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable
-exemptLabels:
- - keepalive
-
-# Set to true to ignore issues in a project (defaults to false)
-exemptProjects: false
-
-# Set to true to ignore issues in a milestone (defaults to false)
-exemptMilestones: false
-
-# Set to true to ignore issues with an assignee (defaults to false)
-exemptAssignees: false
-
-# Label to use when marking as stale
-staleLabel: stale
-
-# Comment to post when marking as stale. Set to `false` to disable
-markComment: false
-
-# Comment to post when removing the stale label.
-# unmarkComment: >
-# Your comment here.
-
-# Comment to post when closing a stale Issue or Pull Request.
-# closeComment: >
-# Your comment here.
-
-# Limit the number of actions per hour, from 1-30. Default is 30
-limitPerRun: 30
-
-# Limit to only `issues` or `pulls`
-only: pulls
-
-# Optionally, specify configuration settings that are specific to just 'issues' or 'pulls':
-# pulls:
-# daysUntilStale: 30
-# markComment: >
-# This pull request has been automatically marked as stale because it has not had
-# recent activity. It will be closed if no further activity occurs. Thank you
-# for your contributions.
-
-# issues:
-# exemptLabels:
-# - confirmed
diff --git a/.github/workflows/buf-lint.yml b/.github/workflows/buf-lint.yml
index acf91ea12a..3f6cf76e16 100644
--- a/.github/workflows/buf-lint.yml
+++ b/.github/workflows/buf-lint.yml
@@ -13,7 +13,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- - uses: bufbuild/buf-setup-action@dde0b9351db90fbf78e345f41a57de8514bf1091 # v1.32.2
+ - uses: bufbuild/buf-setup-action@54abbed4fe8d8d45173eca4798b0c39a53a7b658 # v1.39.0
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
- uses: bufbuild/buf-lint-action@06f9dd823d873146471cfaaf108a993fe00e5325 # v1.1.1
diff --git a/.github/workflows/buf.yml b/.github/workflows/buf.yml
index f52d20785f..632d38cb00 100644
--- a/.github/workflows/buf.yml
+++ b/.github/workflows/buf.yml
@@ -13,7 +13,7 @@ jobs:
if: github.repository_owner == 'prometheus'
steps:
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- - uses: bufbuild/buf-setup-action@dde0b9351db90fbf78e345f41a57de8514bf1091 # v1.32.2
+ - uses: bufbuild/buf-setup-action@54abbed4fe8d8d45173eca4798b0c39a53a7b658 # v1.39.0
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
- uses: bufbuild/buf-lint-action@06f9dd823d873146471cfaaf108a993fe00e5325 # v1.1.1
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 8b3624383c..9e614cb2db 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -11,12 +11,14 @@ jobs:
container:
# Whenever the Go version is updated here, .promu.yml
# should also be updated.
- image: quay.io/prometheus/golang-builder:1.22-base
+ image: quay.io/prometheus/golang-builder:1.23-base
steps:
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- - uses: prometheus/promci@3cb0c3871f223bd5ce1226995bd52ffb314798b6 # v0.1.0
+ - uses: prometheus/promci@468927c440349ab56c4a1aafd453b312841503c2 # v0.4.4
- uses: ./.github/promci/actions/setup_environment
- - run: make GOOPTS=--tags=stringlabels GO_ONLY=1 SKIP_GOLANGCI_LINT=1
+ with:
+ enable_npm: true
+ - run: make GOOPTS=--tags=stringlabels GO_ONLY=1 SKIP_GOLANGCI_LINT=1 test-flags=""
- run: go test --tags=stringlabels ./tsdb/ -test.tsdb-isolation=false
- run: make -C documentation/examples/remote_storage
- run: make -C documentation/examples
@@ -25,10 +27,10 @@ jobs:
name: More Go tests
runs-on: ubuntu-latest
container:
- image: quay.io/prometheus/golang-builder:1.22-base
+ image: quay.io/prometheus/golang-builder:1.23-base
steps:
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- - uses: prometheus/promci@3cb0c3871f223bd5ce1226995bd52ffb314798b6 # v0.1.0
+ - uses: prometheus/promci@468927c440349ab56c4a1aafd453b312841503c2 # v0.4.4
- uses: ./.github/promci/actions/setup_environment
- run: go test --tags=dedupelabels ./...
- run: GOARCH=386 go test ./cmd/prometheus
@@ -39,9 +41,12 @@ jobs:
test_go_oldest:
name: Go tests with previous Go version
runs-on: ubuntu-latest
+ env:
+ # Enforce the Go version.
+ GOTOOLCHAIN: local
container:
# The go version in this image should be N-1 wrt test_go.
- image: quay.io/prometheus/golang-builder:1.21-base
+ image: quay.io/prometheus/golang-builder:1.22-base
steps:
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- run: make build
@@ -54,11 +59,11 @@ jobs:
# Whenever the Go version is updated here, .promu.yml
# should also be updated.
container:
- image: quay.io/prometheus/golang-builder:1.22-base
+ image: quay.io/prometheus/golang-builder:1.23-base
steps:
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- - uses: prometheus/promci@3cb0c3871f223bd5ce1226995bd52ffb314798b6 # v0.1.0
+ - uses: prometheus/promci@468927c440349ab56c4a1aafd453b312841503c2 # v0.4.4
- uses: ./.github/promci/actions/setup_environment
with:
enable_go: false
@@ -75,9 +80,9 @@ jobs:
runs-on: windows-latest
steps:
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- - uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
+ - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
with:
- go-version: 1.22.x
+ go-version: 1.23.x
- run: |
$TestTargets = go list ./... | Where-Object { $_ -NotMatch "(github.com/prometheus/prometheus/discovery.*|github.com/prometheus/prometheus/config|github.com/prometheus/prometheus/web)"}
go test $TestTargets -vet=off -v
@@ -89,7 +94,7 @@ jobs:
# Whenever the Go version is updated here, .promu.yml
# should also be updated.
container:
- image: quay.io/prometheus/golang-builder:1.22-base
+ image: quay.io/prometheus/golang-builder:1.23-base
steps:
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- run: go install ./cmd/promtool/.
@@ -107,6 +112,8 @@ jobs:
if: |
!(github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v2.'))
&&
+ !(github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v3.'))
+ &&
!(github.event_name == 'pull_request' && startsWith(github.event.pull_request.base.ref, 'release-'))
&&
!(github.event_name == 'push' && github.event.ref == 'refs/heads/main')
@@ -115,7 +122,7 @@ jobs:
thread: [ 0, 1, 2 ]
steps:
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- - uses: prometheus/promci@3cb0c3871f223bd5ce1226995bd52ffb314798b6 # v0.1.0
+ - uses: prometheus/promci@468927c440349ab56c4a1aafd453b312841503c2 # v0.4.4
- uses: ./.github/promci/actions/build
with:
promu_opts: "-p linux/amd64 -p windows/amd64 -p linux/arm64 -p darwin/amd64 -p darwin/arm64 -p linux/386"
@@ -127,6 +134,8 @@ jobs:
if: |
(github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v2.'))
||
+ (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v3.'))
+ ||
(github.event_name == 'pull_request' && startsWith(github.event.pull_request.base.ref, 'release-'))
||
(github.event_name == 'push' && github.event.ref == 'refs/heads/main')
@@ -138,11 +147,23 @@ jobs:
# should also be updated.
steps:
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- - uses: prometheus/promci@3cb0c3871f223bd5ce1226995bd52ffb314798b6 # v0.1.0
+ - uses: prometheus/promci@468927c440349ab56c4a1aafd453b312841503c2 # v0.4.4
- uses: ./.github/promci/actions/build
with:
parallelism: 12
thread: ${{ matrix.thread }}
+ build_all_status:
+ name: Report status of build Prometheus for all architectures
+ runs-on: ubuntu-latest
+ needs: [build_all]
+ if: github.event_name == 'pull_request' && startsWith(github.event.pull_request.base.ref, 'release-')
+ steps:
+ - name: Successful build
+ if: ${{ !(contains(needs.*.result, 'failure')) && !(contains(needs.*.result, 'cancelled')) }}
+ run: exit 0
+ - name: Failing or cancelled build
+ if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}
+ run: exit 1
check_generated_parser:
name: Check generated parser
runs-on: ubuntu-latest
@@ -150,10 +171,10 @@ jobs:
- name: Checkout repository
uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- name: Install Go
- uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
+ uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
with:
cache: false
- go-version: 1.22.x
+ go-version: 1.23.x
- name: Run goyacc and check for diff
run: make install-goyacc check-generated-parser
golangci:
@@ -163,18 +184,18 @@ jobs:
- name: Checkout repository
uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- name: Install Go
- uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
+ uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
with:
- go-version: 1.22.x
+ go-version: 1.23.x
- name: Install snmp_exporter/generator dependencies
run: sudo apt-get update && sudo apt-get -y install libsnmp-dev
if: github.repository == 'prometheus/snmp_exporter'
- name: Lint
- uses: golangci/golangci-lint-action@a4f60bb28d35aeee14e6880718e0c85ff1882e64 # v6.0.1
+ uses: golangci/golangci-lint-action@aaa42aa0628b4ae2578232a66b541047968fac86 # v6.1.0
with:
args: --verbose
# Make sure to sync this with Makefile.common and scripts/golangci-lint.yml.
- version: v1.59.1
+ version: v1.60.2
fuzzing:
uses: ./.github/workflows/fuzzing.yml
if: github.event_name == 'pull_request'
@@ -188,7 +209,7 @@ jobs:
if: github.event_name == 'push' && github.event.ref == 'refs/heads/main'
steps:
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- - uses: prometheus/promci@3cb0c3871f223bd5ce1226995bd52ffb314798b6 # v0.1.0
+ - uses: prometheus/promci@468927c440349ab56c4a1aafd453b312841503c2 # v0.4.4
- uses: ./.github/promci/actions/publish_main
with:
docker_hub_login: ${{ secrets.docker_hub_login }}
@@ -199,10 +220,13 @@ jobs:
name: Publish release artefacts
runs-on: ubuntu-latest
needs: [test_ui, test_go, test_go_more, test_go_oldest, test_windows, golangci, codeql, build_all]
- if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v2.')
+ if: |
+ (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v2.'))
+ ||
+ (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v3.'))
steps:
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- - uses: prometheus/promci@3cb0c3871f223bd5ce1226995bd52ffb314798b6 # v0.1.0
+ - uses: prometheus/promci@468927c440349ab56c4a1aafd453b312841503c2 # v0.4.4
- uses: ./.github/promci/actions/publish_release
with:
docker_hub_login: ${{ secrets.docker_hub_login }}
@@ -217,9 +241,9 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- - uses: prometheus/promci@3cb0c3871f223bd5ce1226995bd52ffb314798b6 # v0.1.0
+ - uses: prometheus/promci@468927c440349ab56c4a1aafd453b312841503c2 # v0.4.4
- name: Install nodejs
- uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
+ uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4.0.3
with:
node-version-file: "web/ui/.nvmrc"
registry-url: "https://registry.npmjs.org"
@@ -230,17 +254,26 @@ jobs:
restore-keys: |
${{ runner.os }}-node-
- name: Check libraries version
- if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v2.')
- run: ./scripts/ui_release.sh --check-package "$(echo ${{ github.ref_name }}|sed s/v2/v0/)"
+ if: |
+ (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v2.'))
+ ||
+ (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v3.'))
+ run: ./scripts/ui_release.sh --check-package "$(./scripts/get_module_version.sh ${{ github.ref_name }})"
- name: build
run: make assets
- name: Copy files before publishing libs
run: ./scripts/ui_release.sh --copy
- name: Publish dry-run libraries
- if: "!(github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v2.'))"
+ if: |
+ !(github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v2.'))
+ &&
+ !(github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v3.'))
run: ./scripts/ui_release.sh --publish dry-run
- name: Publish libraries
- if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v2.')
+ if: |
+ (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v2.'))
+ ||
+ (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v3.'))
run: ./scripts/ui_release.sh --publish
env:
# The setup-node action writes an .npmrc file with this env variable
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
index 1ea1f5efae..89aa2ba29b 100644
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@@ -27,12 +27,12 @@ jobs:
uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- name: Initialize CodeQL
- uses: github/codeql-action/init@2e230e8fe0ad3a14a340ad0815ddb96d599d2aff # v3.25.8
+ uses: github/codeql-action/init@4dd16135b69a43b6c8efb853346f8437d92d3c93 # v3.26.6
with:
languages: ${{ matrix.language }}
- name: Autobuild
- uses: github/codeql-action/autobuild@2e230e8fe0ad3a14a340ad0815ddb96d599d2aff # v3.25.8
+ uses: github/codeql-action/autobuild@4dd16135b69a43b6c8efb853346f8437d92d3c93 # v3.26.6
- name: Perform CodeQL Analysis
- uses: github/codeql-action/analyze@2e230e8fe0ad3a14a340ad0815ddb96d599d2aff # v3.25.8
+ uses: github/codeql-action/analyze@4dd16135b69a43b6c8efb853346f8437d92d3c93 # v3.26.6
diff --git a/.github/workflows/fuzzing.yml b/.github/workflows/fuzzing.yml
index dc510e5966..80356e45bf 100644
--- a/.github/workflows/fuzzing.yml
+++ b/.github/workflows/fuzzing.yml
@@ -21,7 +21,7 @@ jobs:
fuzz-seconds: 600
dry-run: false
- name: Upload Crash
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
+ uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
if: failure() && steps.build.outcome == 'success'
with:
name: artifacts
diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml
index 078084888a..7e0ed7dc36 100644
--- a/.github/workflows/scorecards.yml
+++ b/.github/workflows/scorecards.yml
@@ -26,7 +26,7 @@ jobs:
persist-credentials: false
- name: "Run analysis"
- uses: ossf/scorecard-action@dc50aa9510b46c811795eb24b2f1ba02a914e534 # tag=v2.3.3
+ uses: ossf/scorecard-action@62b2cac7ed8198b15735ed49ab1e5cf35480ba46 # tag=v2.4.0
with:
results_file: results.sarif
results_format: sarif
@@ -37,7 +37,7 @@ jobs:
# Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
# format to the repository Actions tab.
- name: "Upload artifact"
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # tag=v4.3.3
+ uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # tag=v4.4.0
with:
name: SARIF file
path: results.sarif
@@ -45,6 +45,6 @@ jobs:
# Upload the results to GitHub's code scanning dashboard.
- name: "Upload to code-scanning"
- uses: github/codeql-action/upload-sarif@2e230e8fe0ad3a14a340ad0815ddb96d599d2aff # tag=v3.25.8
+ uses: github/codeql-action/upload-sarif@4dd16135b69a43b6c8efb853346f8437d92d3c93 # tag=v3.26.6
with:
sarif_file: results.sarif
diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml
new file mode 100644
index 0000000000..d71bcbc9d8
--- /dev/null
+++ b/.github/workflows/stale.yml
@@ -0,0 +1,31 @@
+name: Stale Check
+on:
+ workflow_dispatch: {}
+ schedule:
+ - cron: '16 22 * * *'
+permissions:
+ issues: write
+ pull-requests: write
+jobs:
+ stale:
+ if: github.repository_owner == 'prometheus' || github.repository_owner == 'prometheus-community' # Don't run this workflow on forks.
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/stale@28ca1036281a5e5922ead5184a1bbf96e5fc984e # v9.0.0
+ with:
+ repo-token: ${{ secrets.GITHUB_TOKEN }}
+ # opt out of defaults to avoid marking issues as stale and closing them
+ # https://github.com/actions/stale#days-before-close
+ # https://github.com/actions/stale#days-before-stale
+ days-before-stale: -1
+ days-before-close: -1
+ # Setting it to empty string to skip comments.
+ # https://github.com/actions/stale#stale-pr-message
+ # https://github.com/actions/stale#stale-issue-message
+ stale-pr-message: ''
+ stale-issue-message: ''
+ operations-per-run: 30
+ # override days-before-stale, for only marking the pull requests as stale
+ days-before-pr-stale: 60
+ stale-pr-label: stale
+ exempt-pr-labels: keepalive
diff --git a/.gitignore b/.gitignore
index e85d766b09..0d99305f69 100644
--- a/.gitignore
+++ b/.gitignore
@@ -22,7 +22,7 @@ benchmark.txt
/documentation/examples/remote_storage/example_write_adapter/example_write_adapter
npm_licenses.tar.bz2
-/web/ui/static/react
+/web/ui/static
/vendor
/.build
diff --git a/.golangci.yml b/.golangci.yml
index 026d68a313..303cd33d8b 100644
--- a/.golangci.yml
+++ b/.golangci.yml
@@ -1,12 +1,5 @@
run:
timeout: 15m
- skip-files:
- # Skip autogenerated files.
- - ^.*\.(pb|y)\.go$
- skip-dirs:
- # Copied it from a different source
- - storage/remote/otlptranslator/prometheusremotewrite
- - storage/remote/otlptranslator/prometheus
output:
sort-results: true
@@ -32,8 +25,34 @@ linters:
- loggercheck
issues:
+ max-issues-per-linter: 0
max-same-issues: 0
+ # The default exclusions are too aggressive. For one, they
+ # essentially disable any linting on doc comments. We disable
+ # default exclusions here and add exclusions fitting our codebase
+ # further down.
+ exclude-use-default: false
+ exclude-files:
+ # Skip autogenerated files.
+ - ^.*\.(pb|y)\.go$
+ exclude-dirs:
+ # Copied it from a different source.
+ - storage/remote/otlptranslator/prometheusremotewrite
+ - storage/remote/otlptranslator/prometheus
exclude-rules:
+ - linters:
+ - errcheck
+ # Taken from the default exclusions (that are otherwise disabled above).
+ text: Error return value of .((os\.)?std(out|err)\..*|.*Close|.*Flush|os\.Remove(All)?|.*print(f|ln)?|os\.(Un)?Setenv). is not checked
+ - linters:
+ - govet
+ # We use many Seek methods that do not follow the usual pattern.
+ text: "stdmethods: method Seek.* should have signature Seek"
+ - linters:
+ - revive
+ # We have stopped at some point to write doc comments on exported symbols.
+ # TODO(beorn7): Maybe we should enforce this again? There are ~500 offenders right now.
+ text: exported (.+) should have comment( \(or a comment on this block\))? or be unexported
- linters:
- gocritic
text: "appendAssign"
@@ -94,15 +113,14 @@ linters-settings:
errorf: false
revive:
# By default, revive will enable only the linting rules that are named in the configuration file.
- # So, it's needed to explicitly set in configuration all required rules.
- # The following configuration enables all the rules from the defaults.toml
- # https://github.com/mgechev/revive/blob/master/defaults.toml
+ # So, it's needed to explicitly enable all required rules here.
rules:
# https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md
- name: blank-imports
+ - name: comment-spacings
- name: context-as-argument
arguments:
- # allow functions with test or bench signatures
+ # Allow functions with test or bench signatures.
- allowTypesBefore: "*testing.T,testing.TB"
- name: context-keys-type
- name: dot-imports
@@ -118,6 +136,8 @@ linters-settings:
- name: increment-decrement
- name: indent-error-flow
- name: package-comments
+ # TODO(beorn7): Currently, we have a lot of missing package doc comments. Maybe we should have them.
+ disabled: true
- name: range
- name: receiver-naming
- name: redefines-builtin-id
diff --git a/.promu.yml b/.promu.yml
index 0aa51d6d31..6feaa6ef64 100644
--- a/.promu.yml
+++ b/.promu.yml
@@ -1,7 +1,7 @@
go:
# Whenever the Go version is updated here,
# .github/workflows should also be updated.
- version: 1.22
+ version: 1.23
repository:
path: github.com/prometheus/prometheus
build:
@@ -28,8 +28,6 @@ tarball:
# Whenever there are new files to include in the tarball,
# remember to make sure the new files will be generated after `make build`.
files:
- - consoles
- - console_libraries
- documentation/examples/prometheus.yml
- LICENSE
- NOTICE
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7e47999934..7fbdadfa62 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,119 @@
## unreleased
+* [CHANGE] `holt_winters` is now called `double_exponential_smoothing` and moves behind the [experimental-promql-functions feature flag](https://prometheus.io/docs/prometheus/latest/feature_flags/#experimental-promql-functions). #14930
+* [BUGFIX] PromQL: Only return "possible non-counter" annotation when `rate` returns points. #14910
+
+## 3.0.0-beta.0 / 2024-09-05
+
+Release 3.0.0-beta.0 includes new features such as a brand new UI and UTF-8 support enabled by default. As a new major version, several breaking changes are introduced. The breaking changes are mainly around the removal of deprecated feature flags and CLI arguments, and the full list can be found below. Most users should be able to try this release out of the box without any configuration changes.
+
+As is traditional with a beta release, we do **not** recommend users install 3.0.0-beta on critical production systems, but we do want everyone to test it out and find bugs.
+
+* [CHANGE] UI: The old web UI has been replaced by a completely new one that is less cluttered and adds a few new features (PromLens-style tree view, better metrics explorer, "Explain" tab). However, it is still missing some features of the old UI (notably, exemplar display and heatmaps). To switch back to the old UI, you can use the feature flag `--enable-feature=old-ui` for the time being. #14872
+* [CHANGE] PromQL: Range selectors and the lookback delta are now left-open, i.e. a sample coinciding with the lower time limit is excluded rather than included. #13904
+* [CHANGE] Kubernetes SD: Remove support for `discovery.k8s.io/v1beta1` API version of EndpointSlice. This version is no longer served as of Kubernetes v1.25. #14365
+* [CHANGE] Kubernetes SD: Remove support for `networking.k8s.io/v1beta1` API version of Ingress. This version is no longer served as of Kubernetes v1.22. #14365
+* [CHANGE] UTF-8: Enable UTF-8 support by default. Prometheus now allows all UTF-8 characters in metric and label names. The corresponding `utf8-name` feature flag has been removed. #14705
+* [CHANGE] Console: Remove example files for the console feature. Users can continue using the console feature by supplying their own JavaScript and templates. #14807
+* [CHANGE] SD: Enable the new service discovery manager by default. This SD manager does not restart unchanged discoveries upon reloading. This makes reloads faster and reduces pressure on service discoveries' sources. The corresponding `new-service-discovery-manager` feature flag has been removed. #14770
+* [CHANGE] Agent mode has been promoted to stable. The feature flag `agent` has been removed. To run Prometheus in Agent mode, use the new `--agent` cmdline arg instead. #14747
+* [CHANGE] Remove deprecated `remote-write-receiver`,`promql-at-modifier`, and `promql-negative-offset` feature flags. #13456, #14526
+* [CHANGE] Remove deprecated `storage.tsdb.allow-overlapping-blocks`, `alertmanager.timeout`, and `storage.tsdb.retention` flags. #14640, #14643
+* [ENHANCEMENT] Move AM discovery page from "Monitoring status" to "Server status". #14875
+* [BUGFIX] Scrape: Do not override target parameter labels with config params. #11029
+
+## 2.55.0-rc.0 / 2024-09-20
+
+* [FEATURE] Support UTF-8 characters in label names - feature flag `utf8-names`. #14482, #14880, #14736, #14727
+* [FEATURE] Support config reload automatically - feature flag `auto-reload-config`. #14769
+* [FEATURE] Scraping: Add the ability to set custom `http_headers` in config. #14817
+* [FEATURE] Scraping: Support feature flag `created-timestamp-zero-ingestion` in OpenMetrics. #14356, #14815
+* [FEATURE] Scraping: `scrape_failure_log_file` option to log failures to a file. #14734
+* [FEATURE] OTLP receiver: Optional promotion of resource attributes to series labels. #14200
+* [FEATURE] Remote-Write: Support Google Cloud Monitoring authorization. #14346
+* [FEATURE] Promtool: `tsdb create-blocks` new option to add labels. #14403
+* [FEATURE] Promtool: `promtool test` adds `--junit` flag to format results. #14506
+* [ENHANCEMENT] OTLP receiver: Warn on exponential histograms with zero count and non-zero sum. #14706
+* [ENHANCEMENT] OTLP receiver: Interrupt translation on context cancellation/timeout. #14612
+* [ENHANCEMENT] Remote Read client: Enable streaming remote read if the server supports it. #11379
+* [ENHANCEMENT] Remote-Write: Don't reshard if we haven't successfully sent a sample since last update. #14450
+* [ENHANCEMENT] PromQL: Delay deletion of `__name__` label to the end of the query evaluation. This is **experimental** and enabled under the feature-flag `promql-delayed-name-removal`. #14477
+* [ENHANCEMENT] PromQL: Experimental `sort_by_label` and `sort_by_label_desc` sort by all labels when label is equal. #14655
+* [ENHANCEMENT] PromQL: Clarify error message logged when Go runtime panic occurs during query evaluation. #14621
+* [ENHANCEMENT] PromQL: Use Kahan summation for better accuracy in `avg` and `avg_over_time`. #14413
+* [ENHANCEMENT] Tracing: Improve PromQL tracing, including showing the operation performed for aggregates, operators, and calls. #14816
+* [ENHANCEMENT] API: Support multiple listening addresses. #14665
+* [ENHANCEMENT] TSDB: Backward compatibility with upcoming index v3. #14934
+* [PERF] TSDB: Query in-order and out-of-order series together. #14354, #14693, #14714, #14831, #14874, #14948
+* [PERF] TSDB: Streamline reading of overlapping out-of-order head chunks. #14729
+* [BUGFIX] SD: Fix dropping targets (with feature flag `new-service-discovery-manager`). #13147
+* [BUGFIX] SD: Stop storing stale targets (with feature flag `new-service-discovery-manager`). #13622
+* [BUGFIX] Scraping: exemplars could be dropped in protobuf scraping. #14810
+* [BUGFIX] Remote-Write: fix metadata sending for experimental Remote-Write V2. #14766
+* [BUGFIX] Remote-Write: Return 4xx not 5xx when timeseries has duplicate label. #14716
+* [BUGFIX] Experimental Native Histograms: many fixes for incorrect results, panics, warnings. #14513, #14575, #14598, #14609, #14611, #14771, #14821
+* [BUGFIX] TSDB: Only count unknown record types in `record_decode_failures_total` metric. #14042
+
+## 2.54.1 / 2024-08-27
+
+* [BUGFIX] Scraping: allow multiple samples on same series, with explicit timestamps. #14685
+* [BUGFIX] Docker SD: fix crash in `match_first_network` mode when container is reconnected to a new network. #14654
+* [BUGFIX] PromQL: fix experimental native histograms getting corrupted due to vector selector bug in range queries. #14538
+* [BUGFIX] PromQL: fix experimental native histogram counter reset detection on stale samples. #14514
+* [BUGFIX] PromQL: fix native histograms getting corrupted due to vector selector bug in range queries. #14605
+
+## 2.54.0 / 2024-08-09
+
+Release 2.54 brings a release candidate of a major new version of [Remote Write: 2.0](https://prometheus.io/docs/specs/remote_write_spec_2_0/).
+This is experimental at this time and may still change.
+Remote-write v2 is enabled by default, but can be disabled via feature-flag `web.remote-write-receiver.accepted-protobuf-messages`.
+
+* [CHANGE] Remote-Write: `highest_timestamp_in_seconds` and `queue_highest_sent_timestamp_seconds` metrics now initialized to 0. #14437
+* [CHANGE] API: Split warnings from info annotations in API response. #14327
+* [FEATURE] Remote-Write: Version 2.0 experimental, plus metadata in WAL via feature flag `metadata-wal-records` (defaults on). #14395,#14427,#14444
+* [FEATURE] PromQL: add limitk() and limit_ratio() aggregation operators. #12503
+* [ENHANCEMENT] PromQL: Accept underscores in literal numbers, e.g. 1_000_000 for 1 million. #12821
+* [ENHANCEMENT] PromQL: float literal numbers and durations are now interchangeable (experimental). Example: `time() - my_timestamp > 10m`. #9138
+* [ENHANCEMENT] PromQL: use Kahan summation for sum(). #14074,#14362
+* [ENHANCEMENT] PromQL (experimental native histograms): Optimize `histogram_count` and `histogram_sum` functions. #14097
+* [ENHANCEMENT] TSDB: Better support for out-of-order experimental native histogram samples. #14438
+* [ENHANCEMENT] TSDB: Optimise seek within index. #14393
+* [ENHANCEMENT] TSDB: Optimise deletion of stale series. #14307
+* [ENHANCEMENT] TSDB: Reduce locking to optimise adding and removing series. #13286,#14286
+* [ENHANCEMENT] TSDB: Small optimisation: streamline special handling for out-of-order data. #14396,#14584
+* [ENHANCEMENT] Regexps: Optimize patterns with multiple prefixes. #13843,#14368
+* [ENHANCEMENT] Regexps: Optimize patterns containing multiple literal strings. #14173
+* [ENHANCEMENT] AWS SD: expose Primary IPv6 addresses as __meta_ec2_primary_ipv6_addresses. #14156
+* [ENHANCEMENT] Docker SD: add MatchFirstNetwork for containers with multiple networks. #10490
+* [ENHANCEMENT] OpenStack SD: Use `flavor.original_name` if available. #14312
+* [ENHANCEMENT] UI (experimental native histograms): more accurate representation. #13680,#14430
+* [ENHANCEMENT] Agent: `out_of_order_time_window` config option now applies to agent. #14094
+* [ENHANCEMENT] Notifier: Send any outstanding Alertmanager notifications when shutting down. #14290
+* [ENHANCEMENT] Rules: Add label-matcher support to Rules API. #10194
+* [ENHANCEMENT] HTTP API: Add url to message logged on error while sending response. #14209
+* [BUGFIX] TSDB: Exclude OOO chunks mapped after compaction starts (introduced by #14396). #14584
+* [BUGFIX] CLI: escape `|` characters when generating docs. #14420
+* [BUGFIX] PromQL (experimental native histograms): Fix some binary operators between native histogram values. #14454
+* [BUGFIX] TSDB: LabelNames API could fail during compaction. #14279
+* [BUGFIX] TSDB: Fix rare issue where pending OOO read can be left dangling if creating querier fails. #14341
+* [BUGFIX] TSDB: fix check for context cancellation in LabelNamesFor. #14302
+* [BUGFIX] Rules: Fix rare panic on reload. #14366
+* [BUGFIX] Config: In YAML marshalling, do not output a regexp field if it was never set. #14004
+* [BUGFIX] Remote-Write: reject samples with future timestamps. #14304
+* [BUGFIX] Remote-Write: Fix data corruption in remote write if max_sample_age is applied. #14078
+* [BUGFIX] Notifier: Fix Alertmanager discovery not updating under heavy load. #14174
+* [BUGFIX] Regexes: some Unicode characters were not matched by case-insensitive comparison. #14170,#14299
+* [BUGFIX] Remote-Read: Resolve occasional segmentation fault on query. #14515
+
+## 2.53.1 / 2024-07-10
+
+Fix a bug which would drop samples in remote-write if the sending flow stalled
+for longer than it takes to write one "WAL segment". How long this takes depends on the size
+of your Prometheus; as a rough guide with 10 million series it is about 2-3 minutes.
+
+* [BUGFIX] Remote-write: stop dropping samples in catch-up #14446
+
## 2.53.0 / 2024-06-16
This release changes the default for GOGC, the Go runtime control for the trade-off between excess memory use and CPU usage. We have found that Prometheus operates with minimal additional CPU usage, but greatly reduced memory by adjusting the upstream Go default from 100 to 75.
@@ -40,7 +153,7 @@ This release changes the default for GOGC, the Go runtime control for the trade-
* [ENHANCEMENT] TSDB: Pause regular block compactions if the head needs to be compacted (prioritize head as it increases memory consumption). #13754
* [ENHANCEMENT] Observability: Improved logging during signal handling termination. #13772
* [ENHANCEMENT] Observability: All log lines for drop series use "num_dropped" key consistently. #13823
-* [ENHANCEMENT] Observability: Log chunk snapshot and mmaped chunk replay duration during WAL replay. #13838
+* [ENHANCEMENT] Observability: Log chunk snapshot and mmapped chunk replay duration during WAL replay. #13838
* [ENHANCEMENT] Observability: Log if the block is being created from WBL during compaction. #13846
* [BUGFIX] PromQL: Fix inaccurate sample number statistic when querying histograms. #13667
* [BUGFIX] PromQL: Fix `histogram_stddev` and `histogram_stdvar` for cases where the histogram has negative buckets. #13852
@@ -577,7 +690,7 @@ The binaries published with this release are built with Go1.17.8 to avoid [CVE-2
## 2.33.0 / 2022-01-29
-* [CHANGE] PromQL: Promote negative offset and `@` modifer to stable features. #10121
+* [CHANGE] PromQL: Promote negative offset and `@` modifier to stable features. #10121
* [CHANGE] Web: Promote remote-write-receiver to stable. #10119
* [FEATURE] Config: Add `stripPort` template function. #10002
* [FEATURE] Promtool: Add cardinality analysis to `check metrics`, enabled by flag `--extended`. #10045
@@ -814,7 +927,7 @@ This vulnerability has been reported by Aaron Devaney from MDSec.
* [ENHANCEMENT] Templating: Enable parsing strings in `humanize` functions. #8682
* [BUGFIX] UI: Provide errors instead of blank page on TSDB Status Page. #8654 #8659
* [BUGFIX] TSDB: Do not panic when writing very large records to the WAL. #8790
-* [BUGFIX] TSDB: Avoid panic when mmaped memory is referenced after the file is closed. #8723
+* [BUGFIX] TSDB: Avoid panic when mmapped memory is referenced after the file is closed. #8723
* [BUGFIX] Scaleway Discovery: Fix nil pointer dereference. #8737
* [BUGFIX] Consul Discovery: Restart no longer required after config update with no targets. #8766
@@ -1740,7 +1853,7 @@ information, read the announcement blog post and migration guide.
## 1.7.0 / 2017-06-06
* [CHANGE] Compress remote storage requests and responses with unframed/raw snappy.
-* [CHANGE] Properly ellide secrets in config.
+* [CHANGE] Properly elide secrets in config.
* [FEATURE] Add OpenStack service discovery.
* [FEATURE] Add ability to limit Kubernetes service discovery to certain namespaces.
* [FEATURE] Add metric for discovered number of Alertmanagers.
diff --git a/Dockerfile b/Dockerfile
index b47f77dcd6..b96b3b765d 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -8,21 +8,16 @@ ARG OS="linux"
COPY .build/${OS}-${ARCH}/prometheus /bin/prometheus
COPY .build/${OS}-${ARCH}/promtool /bin/promtool
COPY documentation/examples/prometheus.yml /etc/prometheus/prometheus.yml
-COPY console_libraries/ /usr/share/prometheus/console_libraries/
-COPY consoles/ /usr/share/prometheus/consoles/
COPY LICENSE /LICENSE
COPY NOTICE /NOTICE
COPY npm_licenses.tar.bz2 /npm_licenses.tar.bz2
WORKDIR /prometheus
-RUN ln -s /usr/share/prometheus/console_libraries /usr/share/prometheus/consoles/ /etc/prometheus/ && \
- chown -R nobody:nobody /etc/prometheus /prometheus
+RUN chown -R nobody:nobody /etc/prometheus /prometheus
USER nobody
EXPOSE 9090
VOLUME [ "/prometheus" ]
ENTRYPOINT [ "/bin/prometheus" ]
CMD [ "--config.file=/etc/prometheus/prometheus.yml", \
- "--storage.tsdb.path=/prometheus", \
- "--web.console.libraries=/usr/share/prometheus/console_libraries", \
- "--web.console.templates=/usr/share/prometheus/consoles" ]
+ "--storage.tsdb.path=/prometheus" ]
diff --git a/MAINTAINERS.md b/MAINTAINERS.md
index 3661ddaa0a..7f4153abc1 100644
--- a/MAINTAINERS.md
+++ b/MAINTAINERS.md
@@ -13,13 +13,12 @@ Maintainers for specific parts of the codebase:
* `k8s`: Frederic Branczyk ( / @brancz)
* `documentation`
* `prometheus-mixin`: Matthias Loibl ( / @metalmatze)
-* `model/histogram` and other code related to native histograms: Björn Rabenstein ( / @beorn7),
+* `model/histogram` and other code related to native histograms: Björn Rabenstein ( / @beorn7),
George Krajcsovits ( / @krajorama)
* `storage`
* `remote`: Callum Styan ( / @cstyan), Bartłomiej Płotka ( / @bwplotka), Tom Wilkie (tom.wilkie@gmail.com / @tomwilkie), Nicolás Pazos ( / @npazosmendez), Alex Greenbank ( / @alexgreenbank)
* `otlptranslator`: Arve Knudsen ( / @aknuds1), Jesús Vázquez ( / @jesusvazquez)
* `tsdb`: Ganesh Vernekar ( / @codesome), Bartłomiej Płotka ( / @bwplotka), Jesús Vázquez ( / @jesusvazquez)
- * `agent`: Robert Fratto ( / @rfratto)
* `web`
* `ui`: Julius Volz ( / @juliusv)
* `module`: Augustin Husson ( @nexucis)
diff --git a/Makefile b/Makefile
index f2bb3fcb7a..0b5935de00 100644
--- a/Makefile
+++ b/Makefile
@@ -30,6 +30,11 @@ include Makefile.common
DOCKER_IMAGE_NAME ?= prometheus
+# Only build UI if PREBUILT_ASSETS_STATIC_DIR is not set
+ifdef PREBUILT_ASSETS_STATIC_DIR
+ SKIP_UI_BUILD = true
+endif
+
.PHONY: update-npm-deps
update-npm-deps:
@echo ">> updating npm dependencies"
@@ -42,13 +47,17 @@ upgrade-npm-deps:
.PHONY: ui-bump-version
ui-bump-version:
- version=$$(sed s/2/0/ < VERSION) && ./scripts/ui_release.sh --bump-version "$${version}"
+ version=$$(./scripts/get_module_version.sh) && ./scripts/ui_release.sh --bump-version "$${version}"
cd web/ui && npm install
git add "./web/ui/package-lock.json" "./**/package.json"
.PHONY: ui-install
ui-install:
cd $(UI_PATH) && npm install
+ # The old React app has been separated from the npm workspaces setup to avoid
+ # issues with conflicting dependencies. This is a temporary solution until the
+ # new Mantine-based UI is fully integrated and the old app can be removed.
+ cd $(UI_PATH)/react-app && npm install
.PHONY: ui-build
ui-build:
@@ -65,10 +74,30 @@ ui-test:
.PHONY: ui-lint
ui-lint:
cd $(UI_PATH) && npm run lint
+ # The old React app has been separated from the npm workspaces setup to avoid
+ # issues with conflicting dependencies. This is a temporary solution until the
+ # new Mantine-based UI is fully integrated and the old app can be removed.
+ cd $(UI_PATH)/react-app && npm run lint
.PHONY: assets
+ifndef SKIP_UI_BUILD
assets: ui-install ui-build
+.PHONY: npm_licenses
+npm_licenses: ui-install
+ @echo ">> bundling npm licenses"
+ rm -f $(REACT_APP_NPM_LICENSES_TARBALL) npm_licenses
+ ln -s . npm_licenses
+ find npm_licenses/$(UI_NODE_MODULES_PATH) -iname "license*" | tar cfj $(REACT_APP_NPM_LICENSES_TARBALL) --files-from=-
+ rm -f npm_licenses
+else
+assets:
+ @echo '>> skipping assets build, pre-built assets provided'
+
+npm_licenses:
+ @echo '>> skipping assets npm licenses, pre-built assets provided'
+endif
+
.PHONY: assets-compress
assets-compress: assets
@echo '>> compressing assets'
@@ -117,14 +146,6 @@ else
test: check-generated-parser common-test ui-build-module ui-test ui-lint check-go-mod-version
endif
-.PHONY: npm_licenses
-npm_licenses: ui-install
- @echo ">> bundling npm licenses"
- rm -f $(REACT_APP_NPM_LICENSES_TARBALL) npm_licenses
- ln -s . npm_licenses
- find npm_licenses/$(UI_NODE_MODULES_PATH) -iname "license*" | tar cfj $(REACT_APP_NPM_LICENSES_TARBALL) --files-from=-
- rm -f npm_licenses
-
.PHONY: tarball
tarball: npm_licenses common-tarball
diff --git a/Makefile.common b/Makefile.common
index e3da72ab47..cbb5d86382 100644
--- a/Makefile.common
+++ b/Makefile.common
@@ -61,7 +61,7 @@ PROMU_URL := https://github.com/prometheus/promu/releases/download/v$(PROMU_
SKIP_GOLANGCI_LINT :=
GOLANGCI_LINT :=
GOLANGCI_LINT_OPTS ?=
-GOLANGCI_LINT_VERSION ?= v1.59.1
+GOLANGCI_LINT_VERSION ?= v1.60.2
# golangci-lint only supports linux, darwin and windows platforms on i386/amd64/arm64.
# windows isn't included here because of the path separator being different.
ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux darwin))
@@ -275,3 +275,9 @@ $(1)_precheck:
exit 1; \
fi
endef
+
+govulncheck: install-govulncheck
+ govulncheck ./...
+
+install-govulncheck:
+ command -v govulncheck > /dev/null || go install golang.org/x/vuln/cmd/govulncheck@latest
diff --git a/README.md b/README.md
index cd14ed2ecb..7528147b0e 100644
--- a/README.md
+++ b/README.md
@@ -12,9 +12,10 @@ examples and guides.
[][hub]
[](https://goreportcard.com/report/github.com/prometheus/prometheus)
[](https://bestpractices.coreinfrastructure.org/projects/486)
+[](https://securityscorecards.dev/viewer/?uri=github.com/prometheus/prometheus)
+[](https://clomonitor.io/projects/cncf/prometheus)
[](https://gitpod.io/#https://github.com/prometheus/prometheus)
[](https://bugs.chromium.org/p/oss-fuzz/issues/list?sort=-opened&can=1&q=proj:prometheus)
-[](https://securityscorecards.dev/viewer/?uri=github.com/prometheus/prometheus)
@@ -114,7 +115,7 @@ The Makefile provides several targets:
Prometheus is bundled with many service discovery plugins.
When building Prometheus from source, you can edit the [plugins.yml](./plugins.yml)
-file to disable some service discoveries. The file is a yaml-formated list of go
+file to disable some service discoveries. The file is a yaml-formatted list of go
import path that will be built into the Prometheus binary.
After you have changed the file, you
diff --git a/RELEASE.md b/RELEASE.md
index f9a42be6b8..8e78a6a3ec 100644
--- a/RELEASE.md
+++ b/RELEASE.md
@@ -57,7 +57,9 @@ Release cadence of first pre-releases being cut is 6 weeks.
| v2.50 | 2024-01-16 | Augustin Husson (GitHub: @nexucis) |
| v2.51 | 2024-03-07 | Bryan Boreham (GitHub: @bboreham) |
| v2.52 | 2024-04-22 | Arthur Silva Sens (GitHub: @ArthurSens) |
-| v2.53 | 2024-06-03 | George Krajcsovits (GitHub: @krajorama) |
+| v2.53 LTS | 2024-06-03 | George Krajcsovits (GitHub: @krajorama) |
+| v2.54 | 2024-07-17 | Bryan Boreham (GitHub: @bboreham) |
+| v2.55 | 2024-09-17 | Bryan Boreham (GitHub: @bboreham) |
If you are interested in volunteering please create a pull request against the [prometheus/prometheus](https://github.com/prometheus/prometheus) repository and propose yourself for the release series of your choice.
@@ -186,7 +188,7 @@ the Prometheus server, we use major version zero releases for the libraries.
Tag the new library release via the following commands:
```bash
-tag="v$(sed s/2/0/ < VERSION)"
+tag="v$(./scripts/get_module_version.sh)"
git tag -s "${tag}" -m "${tag}"
git push origin "${tag}"
```
diff --git a/SECURITY-INSIGHTS.yml b/SECURITY-INSIGHTS.yml
new file mode 100644
index 0000000000..009b356214
--- /dev/null
+++ b/SECURITY-INSIGHTS.yml
@@ -0,0 +1,48 @@
+header:
+ schema-version: '1.0.0'
+ expiration-date: '2025-07-30T01:00:00.000Z'
+ last-updated: '2024-07-30'
+ last-reviewed: '2024-07-30'
+ project-url: https://github.com/prometheus/prometheus
+ changelog: https://github.com/prometheus/prometheus/blob/main/CHANGELOG.md
+ license: https://github.com/prometheus/prometheus/blob/main/LICENSE
+project-lifecycle:
+ status: active
+ bug-fixes-only: false
+ core-maintainers:
+ - https://github.com/prometheus/prometheus/blob/main/MAINTAINERS.md
+contribution-policy:
+ accepts-pull-requests: true
+ accepts-automated-pull-requests: true
+dependencies:
+ third-party-packages: true
+ dependencies-lists:
+ - https://github.com/prometheus/prometheus/blob/main/go.mod
+ - https://github.com/prometheus/prometheus/blob/main/web/ui/package.json
+ env-dependencies-policy:
+ policy-url: https://github.com/prometheus/prometheus/blob/main/CONTRIBUTING.md#dependency-management
+distribution-points:
+ - https://github.com/prometheus/prometheus/releases
+documentation:
+ - https://prometheus.io/docs/introduction/overview/
+security-contacts:
+ - type: email
+ value: prometheus-team@googlegroups.com
+security-testing:
+ - tool-type: sca
+ tool-name: Dependabot
+ tool-version: latest
+ integration:
+ ad-hoc: false
+ ci: true
+ before-release: true
+ - tool-type: sast
+ tool-name: CodeQL
+ tool-version: latest
+ integration:
+ ad-hoc: false
+ ci: true
+ before-release: true
+vulnerability-reporting:
+ accepts-vulnerability-reports: true
+ security-policy: https://github.com/prometheus/prometheus/security/policy
diff --git a/VERSION b/VERSION
index 261d95596f..7e9b524994 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.53.0
+3.0.0-beta.0
diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go
index 7544f276a6..e7fd82e6f3 100644
--- a/cmd/prometheus/main.go
+++ b/cmd/prometheus/main.go
@@ -58,8 +58,6 @@ import (
"github.com/prometheus/prometheus/config"
"github.com/prometheus/prometheus/discovery"
- "github.com/prometheus/prometheus/discovery/legacymanager"
- "github.com/prometheus/prometheus/discovery/targetgroup"
"github.com/prometheus/prometheus/model/exemplar"
"github.com/prometheus/prometheus/model/histogram"
"github.com/prometheus/prometheus/model/labels"
@@ -103,6 +101,8 @@ var (
)
func init() {
+ // This can be removed when the default validation scheme in common is updated.
+ model.NameValidationScheme = model.UTF8Validation
prometheus.MustRegister(versioncollector.NewCollector(strings.ReplaceAll(appName, "-", "_")))
var err error
@@ -152,13 +152,16 @@ type flagConfig struct {
queryConcurrency int
queryMaxSamples int
RemoteFlushDeadline model.Duration
+ nameEscapingScheme string
+
+ enableAutoReload bool
+ autoReloadInterval model.Duration
featureList []string
memlimitRatio float64
// These options are extracted from featureList
// for ease of use.
enableExpandExternalLabels bool
- enableNewSDManager bool
enablePerStepStats bool
enableAutoGOMAXPROCS bool
enableAutoGOMEMLIMIT bool
@@ -168,6 +171,8 @@ type flagConfig struct {
corsRegexString string
promlogConfig promlog.Config
+
+ promqlEnableDelayedNameRemoval bool
}
// setFeatureListOptions sets the corresponding options from the featureList.
@@ -176,9 +181,6 @@ func (c *flagConfig) setFeatureListOptions(logger log.Logger) error {
opts := strings.Split(f, ",")
for _, o := range opts {
switch o {
- case "remote-write-receiver":
- c.web.EnableRemoteWriteReceiver = true
- level.Warn(logger).Log("msg", "Remote write receiver enabled via feature flag remote-write-receiver. This is DEPRECATED. Use --web.enable-remote-write-receiver.")
case "otlp-write-receiver":
c.web.EnableOTLPWriteReceiver = true
level.Info(logger).Log("msg", "Experimental OTLP write receiver enabled")
@@ -194,18 +196,21 @@ func (c *flagConfig) setFeatureListOptions(logger log.Logger) error {
case "extra-scrape-metrics":
c.scrape.ExtraMetrics = true
level.Info(logger).Log("msg", "Experimental additional scrape metrics enabled")
- case "new-service-discovery-manager":
- c.enableNewSDManager = true
- level.Info(logger).Log("msg", "Experimental service discovery manager")
- case "agent":
- agentMode = true
- level.Info(logger).Log("msg", "Experimental agent mode enabled.")
+ case "metadata-wal-records":
+ c.scrape.AppendMetadata = true
+ level.Info(logger).Log("msg", "Experimental metadata records in WAL enabled, required for remote write 2.0")
case "promql-per-step-stats":
c.enablePerStepStats = true
level.Info(logger).Log("msg", "Experimental per-step statistics reporting")
case "auto-gomaxprocs":
c.enableAutoGOMAXPROCS = true
level.Info(logger).Log("msg", "Automatically set GOMAXPROCS to match Linux container CPU quota")
+ case "auto-reload-config":
+ c.enableAutoReload = true
+ if s := time.Duration(c.autoReloadInterval).Seconds(); s > 0 && s < 1 {
+ c.autoReloadInterval, _ = model.ParseDuration("1s")
+ }
+ level.Info(logger).Log("msg", fmt.Sprintf("Enabled automatic configuration file reloading. Checking for configuration changes every %s.", c.autoReloadInterval))
case "auto-gomemlimit":
c.enableAutoGOMEMLIMIT = true
level.Info(logger).Log("msg", "Automatically set GOMEMLIMIT to match Linux container or system memory limit")
@@ -225,16 +230,26 @@ func (c *flagConfig) setFeatureListOptions(logger log.Logger) error {
config.DefaultConfig.GlobalConfig.ScrapeProtocols = config.DefaultProtoFirstScrapeProtocols
config.DefaultGlobalConfig.ScrapeProtocols = config.DefaultProtoFirstScrapeProtocols
level.Info(logger).Log("msg", "Experimental native histogram support enabled. Changed default scrape_protocols to prefer PrometheusProto format.", "global.scrape_protocols", fmt.Sprintf("%v", config.DefaultGlobalConfig.ScrapeProtocols))
+ case "ooo-native-histograms":
+ c.tsdb.EnableOOONativeHistograms = true
+ level.Info(logger).Log("msg", "Experimental out-of-order native histogram ingestion enabled. This will only take effect if OutOfOrderTimeWindow is > 0 and if EnableNativeHistograms = true")
case "created-timestamp-zero-ingestion":
c.scrape.EnableCreatedTimestampZeroIngestion = true
// Change relevant global variables. Hacky, but it's hard to pass a new option or default to unmarshallers.
config.DefaultConfig.GlobalConfig.ScrapeProtocols = config.DefaultProtoFirstScrapeProtocols
config.DefaultGlobalConfig.ScrapeProtocols = config.DefaultProtoFirstScrapeProtocols
level.Info(logger).Log("msg", "Experimental created timestamp zero ingestion enabled. Changed default scrape_protocols to prefer PrometheusProto format.", "global.scrape_protocols", fmt.Sprintf("%v", config.DefaultGlobalConfig.ScrapeProtocols))
+ case "delayed-compaction":
+ c.tsdb.EnableDelayedCompaction = true
+ level.Info(logger).Log("msg", "Experimental delayed compaction is enabled.")
+ case "promql-delayed-name-removal":
+ c.promqlEnableDelayedNameRemoval = true
+ level.Info(logger).Log("msg", "Experimental PromQL delayed name removal enabled.")
case "":
continue
- case "promql-at-modifier", "promql-negative-offset":
- level.Warn(logger).Log("msg", "This option for --enable-feature is now permanently enabled and therefore a no-op.", "option", o)
+ case "old-ui":
+ c.web.UseOldUI = true
+ level.Info(logger).Log("msg", "Serving previous version of the Prometheus web UI.")
default:
level.Warn(logger).Log("msg", "Unknown option for --enable-feature", "option", o)
}
@@ -250,11 +265,6 @@ func main() {
runtime.SetMutexProfileFraction(20)
}
- var (
- oldFlagRetentionDuration model.Duration
- newFlagRetentionDuration model.Duration
- )
-
// Unregister the default GoCollector, and reregister with our defaults.
if prometheus.Unregister(collectors.NewGoCollector()) {
prometheus.MustRegister(
@@ -287,8 +297,11 @@ func main() {
a.Flag("config.file", "Prometheus configuration file path.").
Default("prometheus.yml").StringVar(&cfg.configFile)
- a.Flag("web.listen-address", "Address to listen on for UI, API, and telemetry.").
- Default("0.0.0.0:9090").StringVar(&cfg.web.ListenAddress)
+ a.Flag("config.auto-reload-interval", "Specifies the interval for checking and automatically reloading the Prometheus configuration file upon detecting changes.").
+ Default("30s").SetValue(&cfg.autoReloadInterval)
+
+ a.Flag("web.listen-address", "Address to listen on for UI, API, and telemetry. Can be repeated.").
+ Default("0.0.0.0:9090").StringsVar(&cfg.web.ListenAddresses)
a.Flag("auto-gomemlimit.ratio", "The ratio of reserved GOMEMLIMIT memory to the detected maximum container or system memory").
Default("0.9").FloatVar(&cfg.memlimitRatio)
@@ -302,7 +315,7 @@ func main() {
"Maximum duration before timing out read of the request, and closing idle connections.").
Default("5m").SetValue(&cfg.webTimeout)
- a.Flag("web.max-connections", "Maximum number of simultaneous connections.").
+ a.Flag("web.max-connections", "Maximum number of simultaneous connections across all listeners.").
Default("512").IntVar(&cfg.web.MaxConnections)
a.Flag("web.external-url",
@@ -322,9 +335,15 @@ func main() {
a.Flag("web.enable-admin-api", "Enable API endpoints for admin control actions.").
Default("false").BoolVar(&cfg.web.EnableAdminAPI)
+ // TODO(bwplotka): Consider allowing those remote receive flags to be changed in config.
+ // See https://github.com/prometheus/prometheus/issues/14410
a.Flag("web.enable-remote-write-receiver", "Enable API endpoint accepting remote write requests.").
Default("false").BoolVar(&cfg.web.EnableRemoteWriteReceiver)
+ supportedRemoteWriteProtoMsgs := config.RemoteWriteProtoMsgs{config.RemoteWriteProtoMsgV1, config.RemoteWriteProtoMsgV2}
+ a.Flag("web.remote-write-receiver.accepted-protobuf-messages", fmt.Sprintf("List of the remote write protobuf messages to accept when receiving the remote writes. Supported values: %v", supportedRemoteWriteProtoMsgs.String())).
+ Default(supportedRemoteWriteProtoMsgs.Strings()...).SetValue(rwProtoMsgFlagValue(&cfg.web.AcceptRemoteWriteProtoMsgs))
+
a.Flag("web.console.templates", "Path to the console template directory, available at /consoles.").
Default("consoles").StringVar(&cfg.web.ConsoleTemplatesPath)
@@ -355,11 +374,8 @@ func main() {
"Size at which to split the tsdb WAL segment files. Example: 100MB").
Hidden().PlaceHolder("").BytesVar(&cfg.tsdb.WALSegmentSize)
- serverOnlyFlag(a, "storage.tsdb.retention", "[DEPRECATED] How long to retain samples in storage. This flag has been deprecated, use \"storage.tsdb.retention.time\" instead.").
- SetValue(&oldFlagRetentionDuration)
-
- serverOnlyFlag(a, "storage.tsdb.retention.time", "How long to retain samples in storage. When this flag is set it overrides \"storage.tsdb.retention\". If neither this flag nor \"storage.tsdb.retention\" nor \"storage.tsdb.retention.size\" is set, the retention time defaults to "+defaultRetentionString+". Units Supported: y, w, d, h, m, s, ms.").
- SetValue(&newFlagRetentionDuration)
+ serverOnlyFlag(a, "storage.tsdb.retention.time", "How long to retain samples in storage. If neither this flag nor \"storage.tsdb.retention.size\" is set, the retention time defaults to "+defaultRetentionString+". Units Supported: y, w, d, h, m, s, ms.").
+ SetValue(&cfg.tsdb.RetentionDuration)
serverOnlyFlag(a, "storage.tsdb.retention.size", "Maximum number of bytes that can be stored for blocks. A unit is required, supported units: B, KB, MB, GB, TB, PB, EB. Ex: \"512MB\". Based on powers-of-2, so 1KB is 1024B.").
BytesVar(&cfg.tsdb.MaxBytes)
@@ -367,11 +383,6 @@ func main() {
serverOnlyFlag(a, "storage.tsdb.no-lockfile", "Do not create lockfile in data directory.").
Default("false").BoolVar(&cfg.tsdb.NoLockfile)
- // TODO: Remove in Prometheus 3.0.
- var b bool
- serverOnlyFlag(a, "storage.tsdb.allow-overlapping-blocks", "[DEPRECATED] This flag has no effect. Overlapping blocks are enabled by default now.").
- Default("true").Hidden().BoolVar(&b)
-
serverOnlyFlag(a, "storage.tsdb.wal-compression", "Compress the tsdb WAL.").
Hidden().Default("true").BoolVar(&cfg.tsdb.WALCompression)
@@ -448,9 +459,6 @@ func main() {
serverOnlyFlag(a, "alertmanager.drain-notification-queue-on-shutdown", "Send any outstanding Alertmanager notifications when shutting down. If false, any outstanding Alertmanager notifications will be dropped when shutting down.").
Default("true").BoolVar(&cfg.notifier.DrainOnShutdown)
- // TODO: Remove in Prometheus 3.0.
- alertmanagerTimeout := a.Flag("alertmanager.timeout", "[DEPRECATED] This flag has no effect.").Hidden().String()
-
serverOnlyFlag(a, "query.lookback-delta", "The maximum lookback duration for retrieving metrics during expression evaluations and federation.").
Default("5m").SetValue(&cfg.lookbackDelta)
@@ -466,9 +474,11 @@ func main() {
a.Flag("scrape.discovery-reload-interval", "Interval used by scrape manager to throttle target groups updates.").
Hidden().Default("5s").SetValue(&cfg.scrape.DiscoveryReloadInterval)
- a.Flag("enable-feature", "Comma separated feature names to enable. Valid options: agent, auto-gomemlimit, exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-per-step-stats, promql-experimental-functions, remote-write-receiver (DEPRECATED), extra-scrape-metrics, new-service-discovery-manager, auto-gomaxprocs, no-default-scrape-port, native-histograms, otlp-write-receiver, created-timestamp-zero-ingestion, concurrent-rule-eval. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details.").
+ a.Flag("enable-feature", "Comma separated feature names to enable. Valid options: auto-gomemlimit, exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-per-step-stats, promql-experimental-functions, extra-scrape-metrics, auto-gomaxprocs, no-default-scrape-port, native-histograms, otlp-write-receiver, created-timestamp-zero-ingestion, concurrent-rule-eval, delayed-compaction, old-ui. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details.").
Default("").StringsVar(&cfg.featureList)
+ a.Flag("agent", "Run Prometheus in 'Agent mode'.").BoolVar(&agentMode)
+
promlogflag.AddFlags(a, &cfg.promlogConfig)
a.Flag("write-documentation", "Generate command line documentation. Internal use.").Hidden().Action(func(ctx *kingpin.ParseContext) error {
@@ -494,6 +504,15 @@ func main() {
os.Exit(1)
}
+ if cfg.nameEscapingScheme != "" {
+ scheme, err := model.ToEscapingScheme(cfg.nameEscapingScheme)
+ if err != nil {
+ fmt.Fprintf(os.Stderr, `Invalid name escaping scheme: %q; Needs to be one of "values", "underscores", or "dots"`, cfg.nameEscapingScheme)
+ os.Exit(1)
+ }
+ model.NameEscapingScheme = scheme
+ }
+
if agentMode && len(serverOnlyFlags) > 0 {
fmt.Fprintf(os.Stderr, "The following flag(s) can not be used in agent mode: %q", serverOnlyFlags)
os.Exit(3)
@@ -514,7 +533,7 @@ func main() {
localStoragePath = cfg.agentStoragePath
}
- cfg.web.ExternalURL, err = computeExternalURL(cfg.prometheusURL, cfg.web.ListenAddress)
+ cfg.web.ExternalURL, err = computeExternalURL(cfg.prometheusURL, cfg.web.ListenAddresses[0])
if err != nil {
fmt.Fprintln(os.Stderr, fmt.Errorf("parse external URL %q: %w", cfg.prometheusURL, err))
os.Exit(2)
@@ -526,10 +545,6 @@ func main() {
os.Exit(2)
}
- if *alertmanagerTimeout != "" {
- level.Warn(logger).Log("msg", "The flag --alertmanager.timeout has no effect and will be removed in the future.")
- }
-
// Throw error for invalid config before starting other components.
var cfgFile *config.Config
if cfgFile, err = config.LoadFile(cfg.configFile, agentMode, false, log.NewNopLogger()); err != nil {
@@ -576,17 +591,6 @@ func main() {
cfg.web.RoutePrefix = "/" + strings.Trim(cfg.web.RoutePrefix, "/")
if !agentMode {
- // Time retention settings.
- if oldFlagRetentionDuration != 0 {
- level.Warn(logger).Log("deprecation_notice", "'storage.tsdb.retention' flag is deprecated use 'storage.tsdb.retention.time' instead.")
- cfg.tsdb.RetentionDuration = oldFlagRetentionDuration
- }
-
- // When the new flag is set it takes precedence.
- if newFlagRetentionDuration != 0 {
- cfg.tsdb.RetentionDuration = newFlagRetentionDuration
- }
-
if cfg.tsdb.RetentionDuration == 0 && cfg.tsdb.MaxBytes == 0 {
cfg.tsdb.RetentionDuration = defaultRetentionDuration
level.Info(logger).Log("msg", "No time or size retention was set so using the default time retention", "duration", defaultRetentionDuration)
@@ -646,7 +650,7 @@ func main() {
var (
localStorage = &readyStorage{stats: tsdb.NewDBStats()}
scraper = &readyScrapeManager{}
- remoteStorage = remote.NewStorage(log.With(logger, "component", "remote"), prometheus.DefaultRegisterer, localStorage.StartTime, localStoragePath, time.Duration(cfg.RemoteFlushDeadline), scraper)
+ remoteStorage = remote.NewStorage(log.With(logger, "component", "remote"), prometheus.DefaultRegisterer, localStorage.StartTime, localStoragePath, time.Duration(cfg.RemoteFlushDeadline), scraper, cfg.scrape.AppendMetadata)
fanoutStorage = storage.NewFanout(logger, localStorage, remoteStorage)
)
@@ -658,8 +662,8 @@ func main() {
ctxScrape, cancelScrape = context.WithCancel(context.Background())
ctxNotify, cancelNotify = context.WithCancel(context.Background())
- discoveryManagerScrape discoveryManager
- discoveryManagerNotify discoveryManager
+ discoveryManagerScrape *discovery.Manager
+ discoveryManagerNotify *discovery.Manager
)
// Kubernetes client metrics are used by Kubernetes SD.
@@ -679,47 +683,22 @@ func main() {
os.Exit(1)
}
- if cfg.enableNewSDManager {
- {
- discMgr := discovery.NewManager(ctxScrape, log.With(logger, "component", "discovery manager scrape"), prometheus.DefaultRegisterer, sdMetrics, discovery.Name("scrape"))
- if discMgr == nil {
- level.Error(logger).Log("msg", "failed to create a discovery manager scrape")
- os.Exit(1)
- }
- discoveryManagerScrape = discMgr
- }
+ discoveryManagerScrape = discovery.NewManager(ctxScrape, log.With(logger, "component", "discovery manager scrape"), prometheus.DefaultRegisterer, sdMetrics, discovery.Name("scrape"))
+ if discoveryManagerScrape == nil {
+ level.Error(logger).Log("msg", "failed to create a discovery manager scrape")
+ os.Exit(1)
+ }
- {
- discMgr := discovery.NewManager(ctxNotify, log.With(logger, "component", "discovery manager notify"), prometheus.DefaultRegisterer, sdMetrics, discovery.Name("notify"))
- if discMgr == nil {
- level.Error(logger).Log("msg", "failed to create a discovery manager notify")
- os.Exit(1)
- }
- discoveryManagerNotify = discMgr
- }
- } else {
- {
- discMgr := legacymanager.NewManager(ctxScrape, log.With(logger, "component", "discovery manager scrape"), prometheus.DefaultRegisterer, sdMetrics, legacymanager.Name("scrape"))
- if discMgr == nil {
- level.Error(logger).Log("msg", "failed to create a discovery manager scrape")
- os.Exit(1)
- }
- discoveryManagerScrape = discMgr
- }
-
- {
- discMgr := legacymanager.NewManager(ctxNotify, log.With(logger, "component", "discovery manager notify"), prometheus.DefaultRegisterer, sdMetrics, legacymanager.Name("notify"))
- if discMgr == nil {
- level.Error(logger).Log("msg", "failed to create a discovery manager notify")
- os.Exit(1)
- }
- discoveryManagerNotify = discMgr
- }
+ discoveryManagerNotify = discovery.NewManager(ctxNotify, log.With(logger, "component", "discovery manager notify"), prometheus.DefaultRegisterer, sdMetrics, discovery.Name("notify"))
+ if discoveryManagerNotify == nil {
+ level.Error(logger).Log("msg", "failed to create a discovery manager notify")
+ os.Exit(1)
}
scrapeManager, err := scrape.NewManager(
&cfg.scrape,
log.With(logger, "component", "scrape manager"),
+ func(s string) (log.Logger, error) { return logging.NewJSONFileLogger(s) },
fanoutStorage,
prometheus.DefaultRegisterer,
)
@@ -769,9 +748,10 @@ func main() {
NoStepSubqueryIntervalFn: noStepSubqueryInterval.Get,
// EnableAtModifier and EnableNegativeOffset have to be
// always on for regular PromQL as of Prometheus v2.33.
- EnableAtModifier: true,
- EnableNegativeOffset: true,
- EnablePerStepStats: cfg.enablePerStepStats,
+ EnableAtModifier: true,
+ EnableNegativeOffset: true,
+ EnablePerStepStats: cfg.enablePerStepStats,
+ EnableDelayedNameRemoval: cfg.promqlEnableDelayedNameRemoval,
}
queryEngine = promql.NewEngine(opts)
@@ -960,15 +940,21 @@ func main() {
})
}
- listener, err := webHandler.Listener()
+ listeners, err := webHandler.Listeners()
if err != nil {
- level.Error(logger).Log("msg", "Unable to start web listener", "err", err)
+ level.Error(logger).Log("msg", "Unable to start web listeners", "err", err)
+ if err := queryEngine.Close(); err != nil {
+ level.Warn(logger).Log("msg", "Closing query engine failed", "err", err)
+ }
os.Exit(1)
}
err = toolkit_web.Validate(*webConfig)
if err != nil {
level.Error(logger).Log("msg", "Unable to validate web configuration file", "err", err)
+ if err := queryEngine.Close(); err != nil {
+ level.Warn(logger).Log("msg", "Closing query engine failed", "err", err)
+ }
os.Exit(1)
}
@@ -990,11 +976,14 @@ func main() {
case <-cancel:
reloadReady.Close()
}
+ if err := queryEngine.Close(); err != nil {
+ level.Warn(logger).Log("msg", "Closing query engine failed", "err", err)
+ }
return nil
},
func(err error) {
close(cancel)
- webHandler.SetReady(false)
+ webHandler.SetReady(web.Stopping)
},
)
}
@@ -1084,6 +1073,15 @@ func main() {
hup := make(chan os.Signal, 1)
signal.Notify(hup, syscall.SIGHUP)
cancel := make(chan struct{})
+
+ var checksum string
+ if cfg.enableAutoReload {
+ checksum, err = config.GenerateChecksum(cfg.configFile)
+ if err != nil {
+ level.Error(logger).Log("msg", "Failed to generate initial checksum for configuration file", "err", err)
+ }
+ }
+
g.Add(
func() error {
<-reloadReady.C
@@ -1093,6 +1091,12 @@ func main() {
case <-hup:
if err := reloadConfig(cfg.configFile, cfg.enableExpandExternalLabels, cfg.tsdb.EnableExemplarStorage, logger, noStepSubqueryInterval, reloaders...); err != nil {
level.Error(logger).Log("msg", "Error reloading config", "err", err)
+ } else if cfg.enableAutoReload {
+ if currentChecksum, err := config.GenerateChecksum(cfg.configFile); err == nil {
+ checksum = currentChecksum
+ } else {
+ level.Error(logger).Log("msg", "Failed to generate checksum during configuration reload", "err", err)
+ }
}
case rc := <-webHandler.Reload():
if err := reloadConfig(cfg.configFile, cfg.enableExpandExternalLabels, cfg.tsdb.EnableExemplarStorage, logger, noStepSubqueryInterval, reloaders...); err != nil {
@@ -1100,6 +1104,30 @@ func main() {
rc <- err
} else {
rc <- nil
+ if cfg.enableAutoReload {
+ if currentChecksum, err := config.GenerateChecksum(cfg.configFile); err == nil {
+ checksum = currentChecksum
+ } else {
+ level.Error(logger).Log("msg", "Failed to generate checksum during configuration reload", "err", err)
+ }
+ }
+ }
+ case <-time.Tick(time.Duration(cfg.autoReloadInterval)):
+ if !cfg.enableAutoReload {
+ continue
+ }
+ currentChecksum, err := config.GenerateChecksum(cfg.configFile)
+ if err != nil {
+ level.Error(logger).Log("msg", "Failed to generate checksum during configuration reload", "err", err)
+ } else if currentChecksum == checksum {
+ continue
+ }
+ level.Info(logger).Log("msg", "Configuration file change detected, reloading the configuration.")
+
+ if err := reloadConfig(cfg.configFile, cfg.enableExpandExternalLabels, cfg.tsdb.EnableExemplarStorage, logger, noStepSubqueryInterval, reloaders...); err != nil {
+ level.Error(logger).Log("msg", "Error reloading config", "err", err)
+ } else {
+ checksum = currentChecksum
}
case <-cancel:
return nil
@@ -1132,7 +1160,7 @@ func main() {
reloadReady.Close()
- webHandler.SetReady(true)
+ webHandler.SetReady(web.Ready)
level.Info(logger).Log("msg", "Server is ready to receive web requests.")
<-cancel
return nil
@@ -1257,7 +1285,7 @@ func main() {
// Web handler.
g.Add(
func() error {
- if err := webHandler.Run(ctxWeb, listener, *webConfig); err != nil {
+ if err := webHandler.Run(ctxWeb, listeners, *webConfig); err != nil {
return fmt.Errorf("error starting web server: %w", err)
}
return nil
@@ -1706,6 +1734,9 @@ type tsdbOptions struct {
MaxExemplars int64
EnableMemorySnapshotOnShutdown bool
EnableNativeHistograms bool
+ EnableDelayedCompaction bool
+ EnableOverlappingCompaction bool
+ EnableOOONativeHistograms bool
}
func (opts tsdbOptions) ToTSDBOptions() tsdb.Options {
@@ -1725,8 +1756,10 @@ func (opts tsdbOptions) ToTSDBOptions() tsdb.Options {
MaxExemplars: opts.MaxExemplars,
EnableMemorySnapshotOnShutdown: opts.EnableMemorySnapshotOnShutdown,
EnableNativeHistograms: opts.EnableNativeHistograms,
+ EnableOOONativeHistograms: opts.EnableOOONativeHistograms,
OutOfOrderTimeWindow: opts.OutOfOrderTimeWindow,
- EnableOverlappingCompaction: true,
+ EnableDelayedCompaction: opts.EnableDelayedCompaction,
+ EnableOverlappingCompaction: opts.EnableOverlappingCompaction,
}
}
@@ -1759,11 +1792,38 @@ func (opts agentOptions) ToAgentOptions(outOfOrderTimeWindow int64) agent.Option
}
}
-// discoveryManager interfaces the discovery manager. This is used to keep using
-// the manager that restarts SD's on reload for a few releases until we feel
-// the new manager can be enabled for all users.
-type discoveryManager interface {
- ApplyConfig(cfg map[string]discovery.Configs) error
- Run() error
- SyncCh() <-chan map[string][]*targetgroup.Group
+// rwProtoMsgFlagParser is a custom parser for config.RemoteWriteProtoMsg enum.
+type rwProtoMsgFlagParser struct {
+ msgs *[]config.RemoteWriteProtoMsg
+}
+
+func rwProtoMsgFlagValue(msgs *[]config.RemoteWriteProtoMsg) kingpin.Value {
+ return &rwProtoMsgFlagParser{msgs: msgs}
+}
+
+// IsCumulative is used by kingpin to tell if it's an array or not.
+func (p *rwProtoMsgFlagParser) IsCumulative() bool {
+ return true
+}
+
+func (p *rwProtoMsgFlagParser) String() string {
+ ss := make([]string, 0, len(*p.msgs))
+ for _, t := range *p.msgs {
+ ss = append(ss, string(t))
+ }
+ return strings.Join(ss, ",")
+}
+
+func (p *rwProtoMsgFlagParser) Set(opt string) error {
+ t := config.RemoteWriteProtoMsg(opt)
+ if err := t.Validate(); err != nil {
+ return err
+ }
+ for _, prev := range *p.msgs {
+ if prev == t {
+ return fmt.Errorf("duplicated %v flag value, got %v already", t, *p.msgs)
+ }
+ }
+ *p.msgs = append(*p.msgs, t)
+ return nil
}
diff --git a/cmd/prometheus/main_test.go b/cmd/prometheus/main_test.go
index 89c171bb5b..c16864cb8c 100644
--- a/cmd/prometheus/main_test.go
+++ b/cmd/prometheus/main_test.go
@@ -30,16 +30,23 @@ import (
"testing"
"time"
+ "github.com/alecthomas/kingpin/v2"
"github.com/go-kit/log"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/common/model"
"github.com/stretchr/testify/require"
+ "github.com/prometheus/prometheus/config"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/notifier"
"github.com/prometheus/prometheus/rules"
)
+func init() {
+ // This can be removed when the default validation scheme in common is updated.
+ model.NameValidationScheme = model.UTF8Validation
+}
+
const startupTime = 10 * time.Second
var (
@@ -346,7 +353,7 @@ func getCurrentGaugeValuesFor(t *testing.T, reg prometheus.Gatherer, metricNames
}
func TestAgentSuccessfulStartup(t *testing.T) {
- prom := exec.Command(promPath, "-test.main", "--enable-feature=agent", "--web.listen-address=0.0.0.0:0", "--config.file="+agentConfig)
+ prom := exec.Command(promPath, "-test.main", "--agent", "--web.listen-address=0.0.0.0:0", "--config.file="+agentConfig)
require.NoError(t, prom.Start())
actualExitStatus := 0
@@ -364,7 +371,7 @@ func TestAgentSuccessfulStartup(t *testing.T) {
}
func TestAgentFailedStartupWithServerFlag(t *testing.T) {
- prom := exec.Command(promPath, "-test.main", "--enable-feature=agent", "--storage.tsdb.path=.", "--web.listen-address=0.0.0.0:0", "--config.file="+promConfig)
+ prom := exec.Command(promPath, "-test.main", "--agent", "--storage.tsdb.path=.", "--web.listen-address=0.0.0.0:0", "--config.file="+promConfig)
output := bytes.Buffer{}
prom.Stderr = &output
@@ -391,7 +398,7 @@ func TestAgentFailedStartupWithServerFlag(t *testing.T) {
}
func TestAgentFailedStartupWithInvalidConfig(t *testing.T) {
- prom := exec.Command(promPath, "-test.main", "--enable-feature=agent", "--web.listen-address=0.0.0.0:0", "--config.file="+promConfig)
+ prom := exec.Command(promPath, "-test.main", "--agent", "--web.listen-address=0.0.0.0:0", "--config.file="+promConfig)
require.NoError(t, prom.Start())
actualExitStatus := 0
@@ -429,7 +436,7 @@ func TestModeSpecificFlags(t *testing.T) {
args := []string{"-test.main", tc.arg, t.TempDir(), "--web.listen-address=0.0.0.0:0"}
if tc.mode == "agent" {
- args = append(args, "--enable-feature=agent", "--config.file="+agentConfig)
+ args = append(args, "--agent", "--config.file="+agentConfig)
} else {
args = append(args, "--config.file="+promConfig)
}
@@ -499,3 +506,65 @@ func TestDocumentation(t *testing.T) {
require.Equal(t, string(expectedContent), generatedContent, "Generated content does not match documentation. Hint: run `make cli-documentation`.")
}
+
+func TestRwProtoMsgFlagParser(t *testing.T) {
+ defaultOpts := config.RemoteWriteProtoMsgs{
+ config.RemoteWriteProtoMsgV1, config.RemoteWriteProtoMsgV2,
+ }
+
+ for _, tcase := range []struct {
+ args []string
+ expected []config.RemoteWriteProtoMsg
+ expectedErr error
+ }{
+ {
+ args: nil,
+ expected: defaultOpts,
+ },
+ {
+ args: []string{"--test-proto-msgs", "test"},
+ expectedErr: errors.New("unknown remote write protobuf message test, supported: prometheus.WriteRequest, io.prometheus.write.v2.Request"),
+ },
+ {
+ args: []string{"--test-proto-msgs", "io.prometheus.write.v2.Request"},
+ expected: config.RemoteWriteProtoMsgs{config.RemoteWriteProtoMsgV2},
+ },
+ {
+ args: []string{
+ "--test-proto-msgs", "io.prometheus.write.v2.Request",
+ "--test-proto-msgs", "io.prometheus.write.v2.Request",
+ },
+ expectedErr: errors.New("duplicated io.prometheus.write.v2.Request flag value, got [io.prometheus.write.v2.Request] already"),
+ },
+ {
+ args: []string{
+ "--test-proto-msgs", "io.prometheus.write.v2.Request",
+ "--test-proto-msgs", "prometheus.WriteRequest",
+ },
+ expected: config.RemoteWriteProtoMsgs{config.RemoteWriteProtoMsgV2, config.RemoteWriteProtoMsgV1},
+ },
+ {
+ args: []string{
+ "--test-proto-msgs", "io.prometheus.write.v2.Request",
+ "--test-proto-msgs", "prometheus.WriteRequest",
+ "--test-proto-msgs", "io.prometheus.write.v2.Request",
+ },
+ expectedErr: errors.New("duplicated io.prometheus.write.v2.Request flag value, got [io.prometheus.write.v2.Request prometheus.WriteRequest] already"),
+ },
+ } {
+ t.Run(strings.Join(tcase.args, ","), func(t *testing.T) {
+ a := kingpin.New("test", "")
+ var opt []config.RemoteWriteProtoMsg
+ a.Flag("test-proto-msgs", "").Default(defaultOpts.Strings()...).SetValue(rwProtoMsgFlagValue(&opt))
+
+ _, err := a.Parse(tcase.args)
+ if tcase.expectedErr != nil {
+ require.Error(t, err)
+ require.Equal(t, tcase.expectedErr, err)
+ } else {
+ require.NoError(t, err)
+ require.Equal(t, tcase.expected, opt)
+ }
+ })
+ }
+}
diff --git a/cmd/prometheus/scrape_failure_log_test.go b/cmd/prometheus/scrape_failure_log_test.go
new file mode 100644
index 0000000000..8d86d719f9
--- /dev/null
+++ b/cmd/prometheus/scrape_failure_log_test.go
@@ -0,0 +1,193 @@
+// Copyright 2024 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "bytes"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+ "go.uber.org/atomic"
+
+ "github.com/prometheus/prometheus/util/testutil"
+)
+
+func TestScrapeFailureLogFile(t *testing.T) {
+ if testing.Short() {
+ t.Skip("skipping test in short mode.")
+ }
+
+ // Tracks the number of requests made to the mock server.
+ var requestCount atomic.Int32
+
+ // Starts a server that always returns HTTP 500 errors.
+ mockServerAddress := startGarbageServer(t, &requestCount)
+
+ // Create a temporary directory for Prometheus configuration and logs.
+ tempDir := t.TempDir()
+
+ // Define file paths for the scrape failure log and Prometheus configuration.
+ // Like other files, the scrape failure log file should be relative to the
+ // config file. Therefore, we split the name we put in the file and the full
+ // path used to check the content of the file.
+ scrapeFailureLogFileName := "scrape_failure.log"
+ scrapeFailureLogFile := filepath.Join(tempDir, scrapeFailureLogFileName)
+ promConfigFile := filepath.Join(tempDir, "prometheus.yml")
+
+ // Step 1: Set up an initial Prometheus configuration that globally
+ // specifies a scrape failure log file.
+ promConfig := fmt.Sprintf(`
+global:
+ scrape_interval: 500ms
+ scrape_failure_log_file: %s
+
+scrape_configs:
+ - job_name: 'test_job'
+ static_configs:
+ - targets: ['%s']
+`, scrapeFailureLogFileName, mockServerAddress)
+
+ err := os.WriteFile(promConfigFile, []byte(promConfig), 0o644)
+ require.NoError(t, err, "Failed to write Prometheus configuration file")
+
+ // Start Prometheus with the generated configuration and a random port, enabling the lifecycle API.
+ port := testutil.RandomUnprivilegedPort(t)
+ params := []string{
+ "-test.main",
+ "--config.file=" + promConfigFile,
+ "--storage.tsdb.path=" + filepath.Join(tempDir, "data"),
+ fmt.Sprintf("--web.listen-address=127.0.0.1:%d", port),
+ "--web.enable-lifecycle",
+ }
+ prometheusProcess := exec.Command(promPath, params...)
+ prometheusProcess.Stdout = os.Stdout
+ prometheusProcess.Stderr = os.Stderr
+
+ err = prometheusProcess.Start()
+ require.NoError(t, err, "Failed to start Prometheus")
+ defer prometheusProcess.Process.Kill()
+
+ // Wait until the mock server receives at least two requests from Prometheus.
+ require.Eventually(t, func() bool {
+ return requestCount.Load() >= 2
+ }, 30*time.Second, 500*time.Millisecond, "Expected at least two requests to the mock server")
+
+ // Verify that the scrape failures have been logged to the specified file.
+ content, err := os.ReadFile(scrapeFailureLogFile)
+ require.NoError(t, err, "Failed to read scrape failure log")
+ require.Contains(t, string(content), "server returned HTTP status 500 Internal Server Error", "Expected scrape failure log entry not found")
+
+ // Step 2: Update the Prometheus configuration to remove the scrape failure
+ // log file setting.
+ promConfig = fmt.Sprintf(`
+global:
+ scrape_interval: 1s
+
+scrape_configs:
+ - job_name: 'test_job'
+ static_configs:
+ - targets: ['%s']
+`, mockServerAddress)
+
+ err = os.WriteFile(promConfigFile, []byte(promConfig), 0o644)
+ require.NoError(t, err, "Failed to update Prometheus configuration file")
+
+ // Reload Prometheus with the updated configuration.
+ reloadPrometheus(t, port)
+
+ // Count the number of lines in the scrape failure log file before any
+ // further requests.
+ preReloadLogLineCount := countLinesInFile(scrapeFailureLogFile)
+
+ // Wait for at least two more requests to the mock server to ensure
+ // Prometheus continues scraping.
+ requestsBeforeReload := requestCount.Load()
+ require.Eventually(t, func() bool {
+ return requestCount.Load() >= requestsBeforeReload+2
+ }, 30*time.Second, 500*time.Millisecond, "Expected two more requests to the mock server after configuration reload")
+
+ // Ensure that no new lines were added to the scrape failure log file after
+ // the configuration change.
+ require.Equal(t, preReloadLogLineCount, countLinesInFile(scrapeFailureLogFile), "No new lines should be added to the scrape failure log file after removing the log setting")
+
+ // Step 3: Re-add the scrape failure log file setting, but this time under
+ // scrape_configs, and reload Prometheus.
+ promConfig = fmt.Sprintf(`
+global:
+ scrape_interval: 1s
+
+scrape_configs:
+ - job_name: 'test_job'
+ scrape_failure_log_file: %s
+ static_configs:
+ - targets: ['%s']
+`, scrapeFailureLogFileName, mockServerAddress)
+
+ err = os.WriteFile(promConfigFile, []byte(promConfig), 0o644)
+ require.NoError(t, err, "Failed to update Prometheus configuration file")
+
+ // Reload Prometheus with the updated configuration.
+ reloadPrometheus(t, port)
+
+ // Wait for at least two more requests to the mock server and verify that
+ // new log entries are created.
+ postReloadLogLineCount := countLinesInFile(scrapeFailureLogFile)
+ requestsBeforeReAddingLog := requestCount.Load()
+ require.Eventually(t, func() bool {
+ return requestCount.Load() >= requestsBeforeReAddingLog+2
+ }, 30*time.Second, 500*time.Millisecond, "Expected two additional requests after re-adding the log setting")
+
+ // Confirm that new lines were added to the scrape failure log file.
+ require.Greater(t, countLinesInFile(scrapeFailureLogFile), postReloadLogLineCount, "New lines should be added to the scrape failure log file after re-adding the log setting")
+}
+
+// reloadPrometheus sends a reload request to the Prometheus server to apply
+// updated configurations.
+func reloadPrometheus(t *testing.T, port int) {
+ resp, err := http.Post(fmt.Sprintf("http://127.0.0.1:%d/-/reload", port), "", nil)
+ require.NoError(t, err, "Failed to reload Prometheus")
+ require.Equal(t, http.StatusOK, resp.StatusCode, "Unexpected status code when reloading Prometheus")
+}
+
+// startGarbageServer sets up a mock server that returns a 500 Internal Server Error
+// for all requests. It also increments the request count each time it's hit.
+func startGarbageServer(t *testing.T, requestCount *atomic.Int32) string {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ requestCount.Inc()
+ w.WriteHeader(http.StatusInternalServerError)
+ }))
+ t.Cleanup(server.Close)
+
+ parsedURL, err := url.Parse(server.URL)
+ require.NoError(t, err, "Failed to parse mock server URL")
+
+ return parsedURL.Host
+}
+
+// countLinesInFile counts and returns the number of lines in the specified file.
+func countLinesInFile(filePath string) int {
+ data, err := os.ReadFile(filePath)
+ if err != nil {
+ return 0 // Return 0 if the file doesn't exist or can't be read.
+ }
+ return bytes.Count(data, []byte{'\n'})
+}
diff --git a/cmd/promtool/backfill.go b/cmd/promtool/backfill.go
index 400cae421a..16491f0416 100644
--- a/cmd/promtool/backfill.go
+++ b/cmd/promtool/backfill.go
@@ -85,7 +85,7 @@ func getCompatibleBlockDuration(maxBlockDuration int64) int64 {
return blockDuration
}
-func createBlocks(input []byte, mint, maxt, maxBlockDuration int64, maxSamplesInAppender int, outputDir string, humanReadable, quiet bool) (returnErr error) {
+func createBlocks(input []byte, mint, maxt, maxBlockDuration int64, maxSamplesInAppender int, outputDir string, humanReadable, quiet bool, customLabels map[string]string) (returnErr error) {
blockDuration := getCompatibleBlockDuration(maxBlockDuration)
mint = blockDuration * (mint / blockDuration)
@@ -102,6 +102,8 @@ func createBlocks(input []byte, mint, maxt, maxBlockDuration int64, maxSamplesIn
nextSampleTs int64 = math.MaxInt64
)
+ lb := labels.NewBuilder(labels.EmptyLabels())
+
for t := mint; t <= maxt; t += blockDuration {
tsUpper := t + blockDuration
if nextSampleTs != math.MaxInt64 && nextSampleTs >= tsUpper {
@@ -162,7 +164,13 @@ func createBlocks(input []byte, mint, maxt, maxBlockDuration int64, maxSamplesIn
l := labels.Labels{}
p.Metric(&l)
- if _, err := app.Append(0, l, *ts, v); err != nil {
+ lb.Reset(l)
+ for name, value := range customLabels {
+ lb.Set(name, value)
+ }
+ lbls := lb.Labels()
+
+ if _, err := app.Append(0, lbls, *ts, v); err != nil {
return fmt.Errorf("add sample: %w", err)
}
@@ -221,13 +229,13 @@ func createBlocks(input []byte, mint, maxt, maxBlockDuration int64, maxSamplesIn
return nil
}
-func backfill(maxSamplesInAppender int, input []byte, outputDir string, humanReadable, quiet bool, maxBlockDuration time.Duration) (err error) {
+func backfill(maxSamplesInAppender int, input []byte, outputDir string, humanReadable, quiet bool, maxBlockDuration time.Duration, customLabels map[string]string) (err error) {
p := textparse.NewOpenMetricsParser(input, nil) // Don't need a SymbolTable to get max and min timestamps.
maxt, mint, err := getMinAndMaxTimestamps(p)
if err != nil {
return fmt.Errorf("getting min and max timestamp: %w", err)
}
- if err = createBlocks(input, mint, maxt, int64(maxBlockDuration/time.Millisecond), maxSamplesInAppender, outputDir, humanReadable, quiet); err != nil {
+ if err = createBlocks(input, mint, maxt, int64(maxBlockDuration/time.Millisecond), maxSamplesInAppender, outputDir, humanReadable, quiet, customLabels); err != nil {
return fmt.Errorf("block creation: %w", err)
}
return nil
diff --git a/cmd/promtool/backfill_test.go b/cmd/promtool/backfill_test.go
index 32abfa46a8..b818194e86 100644
--- a/cmd/promtool/backfill_test.go
+++ b/cmd/promtool/backfill_test.go
@@ -92,6 +92,7 @@ func TestBackfill(t *testing.T) {
Description string
MaxSamplesInAppender int
MaxBlockDuration time.Duration
+ Labels map[string]string
Expected struct {
MinTime int64
MaxTime int64
@@ -636,6 +637,49 @@ http_requests_total{code="400"} 1024 7199
},
},
},
+ {
+ ToParse: `# HELP http_requests_total The total number of HTTP requests.
+# TYPE http_requests_total counter
+http_requests_total{code="200"} 1 1624463088.000
+http_requests_total{code="200"} 2 1629503088.000
+http_requests_total{code="200"} 3 1629863088.000
+# EOF
+`,
+ IsOk: true,
+ Description: "Sample with external labels.",
+ MaxSamplesInAppender: 5000,
+ MaxBlockDuration: 2048 * time.Hour,
+ Labels: map[string]string{"cluster_id": "123", "org_id": "999"},
+ Expected: struct {
+ MinTime int64
+ MaxTime int64
+ NumBlocks int
+ BlockDuration int64
+ Samples []backfillSample
+ }{
+ MinTime: 1624463088000,
+ MaxTime: 1629863088000,
+ NumBlocks: 2,
+ BlockDuration: int64(1458 * time.Hour / time.Millisecond),
+ Samples: []backfillSample{
+ {
+ Timestamp: 1624463088000,
+ Value: 1,
+ Labels: labels.FromStrings("__name__", "http_requests_total", "code", "200", "cluster_id", "123", "org_id", "999"),
+ },
+ {
+ Timestamp: 1629503088000,
+ Value: 2,
+ Labels: labels.FromStrings("__name__", "http_requests_total", "code", "200", "cluster_id", "123", "org_id", "999"),
+ },
+ {
+ Timestamp: 1629863088000,
+ Value: 3,
+ Labels: labels.FromStrings("__name__", "http_requests_total", "code", "200", "cluster_id", "123", "org_id", "999"),
+ },
+ },
+ },
+ },
{
ToParse: `# HELP rpc_duration_seconds A summary of the RPC duration in seconds.
# TYPE rpc_duration_seconds summary
@@ -689,7 +733,7 @@ after_eof 1 2
outputDir := t.TempDir()
- err := backfill(test.MaxSamplesInAppender, []byte(test.ToParse), outputDir, false, false, test.MaxBlockDuration)
+ err := backfill(test.MaxSamplesInAppender, []byte(test.ToParse), outputDir, false, false, test.MaxBlockDuration, test.Labels)
if !test.IsOk {
require.Error(t, err, test.Description)
diff --git a/cmd/promtool/main.go b/cmd/promtool/main.go
index e1d275e97e..4d4cf6c5db 100644
--- a/cmd/promtool/main.go
+++ b/cmd/promtool/main.go
@@ -62,6 +62,11 @@ import (
"github.com/prometheus/prometheus/util/documentcli"
)
+func init() {
+ // This can be removed when the default validation scheme in common is updated.
+ model.NameValidationScheme = model.UTF8Validation
+}
+
const (
successExitCode = 0
failureExitCode = 1
@@ -204,6 +209,7 @@ func main() {
pushMetricsHeaders := pushMetricsCmd.Flag("header", "Prometheus remote write header.").StringMap()
testCmd := app.Command("test", "Unit testing.")
+ junitOutFile := testCmd.Flag("junit", "File path to store JUnit XML test results.").OpenFile(os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644)
testRulesCmd := testCmd.Command("rules", "Unit tests for rules.")
testRulesRun := testRulesCmd.Flag("run", "If set, will only run test groups whose names match the regular expression. Can be specified multiple times.").Strings()
testRulesFiles := testRulesCmd.Arg(
@@ -235,14 +241,14 @@ func main() {
tsdbDumpCmd := tsdbCmd.Command("dump", "Dump samples from a TSDB.")
dumpPath := tsdbDumpCmd.Arg("db path", "Database path (default is "+defaultDBPath+").").Default(defaultDBPath).String()
- dumpSandboxDirRoot := tsdbDumpCmd.Flag("sandbox-dir-root", "Root directory where a sandbox directory would be created in case WAL replay generates chunks. The sandbox directory is cleaned up at the end.").Default(defaultDBPath).String()
+ dumpSandboxDirRoot := tsdbDumpCmd.Flag("sandbox-dir-root", "Root directory where a sandbox directory will be created, this sandbox is used in case WAL replay generates chunks (default is the database path). The sandbox is cleaned up at the end.").String()
dumpMinTime := tsdbDumpCmd.Flag("min-time", "Minimum timestamp to dump.").Default(strconv.FormatInt(math.MinInt64, 10)).Int64()
dumpMaxTime := tsdbDumpCmd.Flag("max-time", "Maximum timestamp to dump.").Default(strconv.FormatInt(math.MaxInt64, 10)).Int64()
dumpMatch := tsdbDumpCmd.Flag("match", "Series selector. Can be specified multiple times.").Default("{__name__=~'(?s:.*)'}").Strings()
tsdbDumpOpenMetricsCmd := tsdbCmd.Command("dump-openmetrics", "[Experimental] Dump samples from a TSDB into OpenMetrics text format, excluding native histograms and staleness markers, which are not representable in OpenMetrics.")
dumpOpenMetricsPath := tsdbDumpOpenMetricsCmd.Arg("db path", "Database path (default is "+defaultDBPath+").").Default(defaultDBPath).String()
- dumpOpenMetricsSandboxDirRoot := tsdbDumpOpenMetricsCmd.Flag("sandbox-dir-root", "Root directory where a sandbox directory would be created in case WAL replay generates chunks. The sandbox directory is cleaned up at the end.").Default(defaultDBPath).String()
+ dumpOpenMetricsSandboxDirRoot := tsdbDumpOpenMetricsCmd.Flag("sandbox-dir-root", "Root directory where a sandbox directory will be created, this sandbox is used in case WAL replay generates chunks (default is the database path). The sandbox is cleaned up at the end.").String()
dumpOpenMetricsMinTime := tsdbDumpOpenMetricsCmd.Flag("min-time", "Minimum timestamp to dump.").Default(strconv.FormatInt(math.MinInt64, 10)).Int64()
dumpOpenMetricsMaxTime := tsdbDumpOpenMetricsCmd.Flag("max-time", "Maximum timestamp to dump.").Default(strconv.FormatInt(math.MaxInt64, 10)).Int64()
dumpOpenMetricsMatch := tsdbDumpOpenMetricsCmd.Flag("match", "Series selector. Can be specified multiple times.").Default("{__name__=~'(?s:.*)'}").Strings()
@@ -252,6 +258,7 @@ func main() {
importQuiet := importCmd.Flag("quiet", "Do not print created blocks.").Short('q').Bool()
maxBlockDuration := importCmd.Flag("max-block-duration", "Maximum duration created blocks may span. Anything less than 2h is ignored.").Hidden().PlaceHolder("").Duration()
openMetricsImportCmd := importCmd.Command("openmetrics", "Import samples from OpenMetrics input and produce TSDB blocks. Please refer to the storage docs for more details.")
+ openMetricsLabels := openMetricsImportCmd.Flag("label", "Label to attach to metrics. Can be specified multiple times. Example --label=label_name=label_value").StringMap()
importFilePath := openMetricsImportCmd.Arg("input file", "OpenMetrics file to read samples from.").Required().String()
importDBPath := openMetricsImportCmd.Arg("output directory", "Output directory for generated blocks.").Default(defaultDBPath).String()
importRulesCmd := importCmd.Command("rules", "Create blocks of data for new recording rules.")
@@ -323,8 +330,6 @@ func main() {
noDefaultScrapePort = true
case "":
continue
- case "promql-at-modifier", "promql-negative-offset":
- fmt.Printf(" WARNING: Option for --enable-feature is a no-op after promotion to a stable feature: %q\n", o)
default:
fmt.Printf(" WARNING: Unknown option for --enable-feature: %q\n", o)
}
@@ -378,7 +383,11 @@ func main() {
os.Exit(QueryLabels(serverURL, httpRoundTripper, *queryLabelsMatch, *queryLabelsName, *queryLabelsBegin, *queryLabelsEnd, p))
case testRulesCmd.FullCommand():
- os.Exit(RulesUnitTest(
+ results := io.Discard
+ if *junitOutFile != nil {
+ results = *junitOutFile
+ }
+ os.Exit(RulesUnitTestResult(results,
promqltest.LazyLoaderOpts{
EnableAtModifier: true,
EnableNegativeOffset: true,
@@ -403,7 +412,7 @@ func main() {
os.Exit(checkErr(dumpSamples(ctx, *dumpOpenMetricsPath, *dumpOpenMetricsSandboxDirRoot, *dumpOpenMetricsMinTime, *dumpOpenMetricsMaxTime, *dumpOpenMetricsMatch, formatSeriesSetOpenMetrics)))
// TODO(aSquare14): Work on adding support for custom block size.
case openMetricsImportCmd.FullCommand():
- os.Exit(backfillOpenMetrics(*importFilePath, *importDBPath, *importHumanReadable, *importQuiet, *maxBlockDuration))
+ os.Exit(backfillOpenMetrics(*importFilePath, *importDBPath, *importHumanReadable, *importQuiet, *maxBlockDuration, *openMetricsLabels))
case importRulesCmd.FullCommand():
os.Exit(checkErr(importRules(serverURL, httpRoundTripper, *importRulesStart, *importRulesEnd, *importRulesOutputDir, *importRulesEvalInterval, *maxBlockDuration, *importRulesFiles...)))
@@ -466,7 +475,7 @@ func (ls lintConfig) lintDuplicateRules() bool {
return ls.all || ls.duplicateRules
}
-// Check server status - healthy & ready.
+// CheckServerStatus - healthy & ready.
func CheckServerStatus(serverURL *url.URL, checkEndpoint string, roundTripper http.RoundTripper) error {
if serverURL.Scheme == "" {
serverURL.Scheme = "http"
diff --git a/cmd/promtool/main_test.go b/cmd/promtool/main_test.go
index 78500fe937..9d891c32fd 100644
--- a/cmd/promtool/main_test.go
+++ b/cmd/promtool/main_test.go
@@ -31,12 +31,19 @@ import (
"testing"
"time"
+ "github.com/prometheus/common/model"
"github.com/stretchr/testify/require"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/model/rulefmt"
+ "github.com/prometheus/prometheus/promql/promqltest"
)
+func init() {
+ // This can be removed when the default validation scheme in common is updated.
+ model.NameValidationScheme = model.UTF8Validation
+}
+
var promtoolPath = os.Args[0]
func TestMain(m *testing.M) {
@@ -549,3 +556,46 @@ func TestCheckRulesWithRuleFiles(t *testing.T) {
require.Equal(t, lintErrExitCode, exitCode, "")
})
}
+
+func TestTSDBDumpCommand(t *testing.T) {
+ if testing.Short() {
+ t.Skip("skipping test in short mode.")
+ }
+
+ storage := promqltest.LoadedStorage(t, `
+ load 1m
+ metric{foo="bar"} 1 2 3
+ `)
+ t.Cleanup(func() { storage.Close() })
+
+ for _, c := range []struct {
+ name string
+ subCmd string
+ sandboxDirRoot string
+ }{
+ {
+ name: "dump",
+ subCmd: "dump",
+ },
+ {
+ name: "dump with sandbox dir root",
+ subCmd: "dump",
+ sandboxDirRoot: t.TempDir(),
+ },
+ {
+ name: "dump-openmetrics",
+ subCmd: "dump-openmetrics",
+ },
+ {
+ name: "dump-openmetrics with sandbox dir root",
+ subCmd: "dump-openmetrics",
+ sandboxDirRoot: t.TempDir(),
+ },
+ } {
+ t.Run(c.name, func(t *testing.T) {
+ args := []string{"-test.main", "tsdb", c.subCmd, storage.Dir()}
+ cmd := exec.Command(promtoolPath, args...)
+ require.NoError(t, cmd.Run())
+ })
+ }
+}
diff --git a/cmd/promtool/metrics.go b/cmd/promtool/metrics.go
index 46246b672a..4c91d1d6fe 100644
--- a/cmd/promtool/metrics.go
+++ b/cmd/promtool/metrics.go
@@ -31,7 +31,7 @@ import (
"github.com/prometheus/prometheus/util/fmtutil"
)
-// Push metrics to a prometheus remote write (for testing purpose only).
+// PushMetrics to a prometheus remote write (for testing purpose only).
func PushMetrics(url *url.URL, roundTripper http.RoundTripper, headers map[string]string, timeout time.Duration, labels map[string]string, files ...string) int {
addressURL, err := url.Parse(url.String())
if err != nil {
@@ -101,6 +101,7 @@ func PushMetrics(url *url.URL, roundTripper http.RoundTripper, headers map[strin
return successExitCode
}
+// TODO(bwplotka): Add PRW 2.0 support.
func parseAndPushMetrics(client *remote.Client, data []byte, labels map[string]string) bool {
metricsData, err := fmtutil.MetricTextToWriteRequest(bytes.NewReader(data), labels)
if err != nil {
@@ -116,7 +117,7 @@ func parseAndPushMetrics(client *remote.Client, data []byte, labels map[string]s
// Encode the request body into snappy encoding.
compressed := snappy.Encode(nil, raw)
- err = client.Store(context.Background(), compressed, 0)
+ _, err = client.Store(context.Background(), compressed, 0)
if err != nil {
fmt.Fprintln(os.Stderr, " FAILED:", err)
return false
diff --git a/cmd/promtool/testdata/unittest.yml b/cmd/promtool/testdata/unittest.yml
index ff511729ba..e2a8230902 100644
--- a/cmd/promtool/testdata/unittest.yml
+++ b/cmd/promtool/testdata/unittest.yml
@@ -69,13 +69,13 @@ tests:
eval_time: 2m
exp_samples:
- labels: "test_histogram_repeat"
- histogram: "{{count:2 sum:3 buckets:[2]}}"
+ histogram: "{{count:2 sum:3 counter_reset_hint:not_reset buckets:[2]}}"
- expr: test_histogram_increase
eval_time: 2m
exp_samples:
- labels: "test_histogram_increase"
- histogram: "{{count:4 sum:5.6 buckets:[4]}}"
+ histogram: "{{count:4 sum:5.6 counter_reset_hint:not_reset buckets:[4]}}"
# Ensure a value is stale as soon as it is marked as such.
- expr: test_stale
@@ -89,11 +89,11 @@ tests:
# Ensure lookback delta is respected, when a value is missing.
- expr: timestamp(test_missing)
- eval_time: 5m
+ eval_time: 4m59s
exp_samples:
- value: 0
- expr: timestamp(test_missing)
- eval_time: 5m1s
+ eval_time: 5m
exp_samples: []
# Minimal test case to check edge case of a single sample.
@@ -113,7 +113,7 @@ tests:
- expr: count_over_time(fixed_data[1h])
eval_time: 1h
exp_samples:
- - value: 61
+ - value: 60
- expr: timestamp(fixed_data)
eval_time: 1h
exp_samples:
@@ -183,7 +183,7 @@ tests:
- expr: job:test:count_over_time1m
eval_time: 1m
exp_samples:
- - value: 61
+ - value: 60
labels: 'job:test:count_over_time1m{job="test"}'
- expr: timestamp(job:test:count_over_time1m)
eval_time: 1m10s
@@ -194,7 +194,7 @@ tests:
- expr: job:test:count_over_time1m
eval_time: 2m
exp_samples:
- - value: 61
+ - value: 60
labels: 'job:test:count_over_time1m{job="test"}'
- expr: timestamp(job:test:count_over_time1m)
eval_time: 2m59s999ms
diff --git a/cmd/promtool/tsdb.go b/cmd/promtool/tsdb.go
index 2ed7244b1c..e6323d2944 100644
--- a/cmd/promtool/tsdb.go
+++ b/cmd/promtool/tsdb.go
@@ -367,25 +367,25 @@ func printBlocks(blocks []tsdb.BlockReader, writeHeader, humanReadable bool) {
fmt.Fprintf(tw,
"%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\n",
meta.ULID,
- getFormatedTime(meta.MinTime, humanReadable),
- getFormatedTime(meta.MaxTime, humanReadable),
+ getFormattedTime(meta.MinTime, humanReadable),
+ getFormattedTime(meta.MaxTime, humanReadable),
time.Duration(meta.MaxTime-meta.MinTime)*time.Millisecond,
meta.Stats.NumSamples,
meta.Stats.NumChunks,
meta.Stats.NumSeries,
- getFormatedBytes(b.Size(), humanReadable),
+ getFormattedBytes(b.Size(), humanReadable),
)
}
}
-func getFormatedTime(timestamp int64, humanReadable bool) string {
+func getFormattedTime(timestamp int64, humanReadable bool) string {
if humanReadable {
return time.Unix(timestamp/1000, 0).UTC().String()
}
return strconv.FormatInt(timestamp, 10)
}
-func getFormatedBytes(bytes int64, humanReadable bool) string {
+func getFormattedBytes(bytes int64, humanReadable bool) string {
if humanReadable {
return units.Base2Bytes(bytes).String()
}
@@ -823,7 +823,7 @@ func checkErr(err error) int {
return 0
}
-func backfillOpenMetrics(path, outputDir string, humanReadable, quiet bool, maxBlockDuration time.Duration) int {
+func backfillOpenMetrics(path, outputDir string, humanReadable, quiet bool, maxBlockDuration time.Duration, customLabels map[string]string) int {
inputFile, err := fileutil.OpenMmapFile(path)
if err != nil {
return checkErr(err)
@@ -834,7 +834,7 @@ func backfillOpenMetrics(path, outputDir string, humanReadable, quiet bool, maxB
return checkErr(fmt.Errorf("create output dir: %w", err))
}
- return checkErr(backfill(5000, inputFile.Bytes(), outputDir, humanReadable, quiet, maxBlockDuration))
+ return checkErr(backfill(5000, inputFile.Bytes(), outputDir, humanReadable, quiet, maxBlockDuration, customLabels))
}
func displayHistogram(dataType string, datas []int, total int) {
@@ -866,16 +866,16 @@ func displayHistogram(dataType string, datas []int, total int) {
fmt.Println()
}
-func generateBucket(min, max int) (start, end, step int) {
- s := (max - min) / 10
+func generateBucket(minVal, maxVal int) (start, end, step int) {
+ s := (maxVal - minVal) / 10
step = 10
for step < s && step <= 10000 {
step *= 10
}
- start = min - min%step
- end = max - max%step + step
+ start = minVal - minVal%step
+ end = maxVal - maxVal%step + step
return
}
diff --git a/cmd/promtool/tsdb_test.go b/cmd/promtool/tsdb_test.go
index 75089b168b..90192e31ab 100644
--- a/cmd/promtool/tsdb_test.go
+++ b/cmd/promtool/tsdb_test.go
@@ -20,6 +20,7 @@ import (
"math"
"os"
"runtime"
+ "slices"
"strings"
"testing"
"time"
@@ -54,7 +55,7 @@ func TestGenerateBucket(t *testing.T) {
}
// getDumpedSamples dumps samples and returns them.
-func getDumpedSamples(t *testing.T, path string, mint, maxt int64, match []string, formatter SeriesSetFormatter) string {
+func getDumpedSamples(t *testing.T, databasePath, sandboxDirRoot string, mint, maxt int64, match []string, formatter SeriesSetFormatter) string {
t.Helper()
oldStdout := os.Stdout
@@ -63,8 +64,8 @@ func getDumpedSamples(t *testing.T, path string, mint, maxt int64, match []strin
err := dumpSamples(
context.Background(),
- path,
- t.TempDir(),
+ databasePath,
+ sandboxDirRoot,
mint,
maxt,
match,
@@ -95,13 +96,15 @@ func TestTSDBDump(t *testing.T) {
heavy_metric{foo="bar"} 5 4 3 2 1
heavy_metric{foo="foo"} 5 4 3 2 1
`)
+ t.Cleanup(func() { storage.Close() })
tests := []struct {
- name string
- mint int64
- maxt int64
- match []string
- expectedDump string
+ name string
+ mint int64
+ maxt int64
+ sandboxDirRoot string
+ match []string
+ expectedDump string
}{
{
name: "default match",
@@ -110,6 +113,14 @@ func TestTSDBDump(t *testing.T) {
match: []string{"{__name__=~'(?s:.*)'}"},
expectedDump: "testdata/dump-test-1.prom",
},
+ {
+ name: "default match with sandbox dir root set",
+ mint: math.MinInt64,
+ maxt: math.MaxInt64,
+ sandboxDirRoot: t.TempDir(),
+ match: []string{"{__name__=~'(?s:.*)'}"},
+ expectedDump: "testdata/dump-test-1.prom",
+ },
{
name: "same matcher twice",
mint: math.MinInt64,
@@ -148,28 +159,51 @@ func TestTSDBDump(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- dumpedMetrics := getDumpedSamples(t, storage.Dir(), tt.mint, tt.maxt, tt.match, formatSeriesSet)
+ dumpedMetrics := getDumpedSamples(t, storage.Dir(), tt.sandboxDirRoot, tt.mint, tt.maxt, tt.match, formatSeriesSet)
expectedMetrics, err := os.ReadFile(tt.expectedDump)
require.NoError(t, err)
expectedMetrics = normalizeNewLine(expectedMetrics)
- // even though in case of one matcher samples are not sorted, the order in the cases above should stay the same.
- require.Equal(t, string(expectedMetrics), dumpedMetrics)
+ // Sort both, because Prometheus does not guarantee the output order.
+ require.Equal(t, sortLines(string(expectedMetrics)), sortLines(dumpedMetrics))
})
}
}
+func sortLines(buf string) string {
+ lines := strings.Split(buf, "\n")
+ slices.Sort(lines)
+ return strings.Join(lines, "\n")
+}
+
func TestTSDBDumpOpenMetrics(t *testing.T) {
storage := promqltest.LoadedStorage(t, `
load 1m
my_counter{foo="bar", baz="abc"} 1 2 3 4 5
my_gauge{bar="foo", abc="baz"} 9 8 0 4 7
`)
+ t.Cleanup(func() { storage.Close() })
- expectedMetrics, err := os.ReadFile("testdata/dump-openmetrics-test.prom")
- require.NoError(t, err)
- expectedMetrics = normalizeNewLine(expectedMetrics)
- dumpedMetrics := getDumpedSamples(t, storage.Dir(), math.MinInt64, math.MaxInt64, []string{"{__name__=~'(?s:.*)'}"}, formatSeriesSetOpenMetrics)
- require.Equal(t, string(expectedMetrics), dumpedMetrics)
+ tests := []struct {
+ name string
+ sandboxDirRoot string
+ }{
+ {
+ name: "default match",
+ },
+ {
+ name: "default match with sandbox dir root set",
+ sandboxDirRoot: t.TempDir(),
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ expectedMetrics, err := os.ReadFile("testdata/dump-openmetrics-test.prom")
+ require.NoError(t, err)
+ expectedMetrics = normalizeNewLine(expectedMetrics)
+ dumpedMetrics := getDumpedSamples(t, storage.Dir(), tt.sandboxDirRoot, math.MinInt64, math.MaxInt64, []string{"{__name__=~'(?s:.*)'}"}, formatSeriesSetOpenMetrics)
+ require.Equal(t, sortLines(string(expectedMetrics)), sortLines(dumpedMetrics))
+ })
+ }
}
func TestTSDBDumpOpenMetricsRoundTrip(t *testing.T) {
@@ -179,7 +213,7 @@ func TestTSDBDumpOpenMetricsRoundTrip(t *testing.T) {
dbDir := t.TempDir()
// Import samples from OM format
- err = backfill(5000, initialMetrics, dbDir, false, false, 2*time.Hour)
+ err = backfill(5000, initialMetrics, dbDir, false, false, 2*time.Hour, map[string]string{})
require.NoError(t, err)
db, err := tsdb.Open(dbDir, nil, nil, tsdb.DefaultOptions(), nil)
require.NoError(t, err)
@@ -188,7 +222,7 @@ func TestTSDBDumpOpenMetricsRoundTrip(t *testing.T) {
})
// Dump the blocks into OM format
- dumpedMetrics := getDumpedSamples(t, dbDir, math.MinInt64, math.MaxInt64, []string{"{__name__=~'(?s:.*)'}"}, formatSeriesSetOpenMetrics)
+ dumpedMetrics := getDumpedSamples(t, dbDir, "", math.MinInt64, math.MaxInt64, []string{"{__name__=~'(?s:.*)'}"}, formatSeriesSetOpenMetrics)
// Should get back the initial metrics.
require.Equal(t, string(initialMetrics), dumpedMetrics)
diff --git a/cmd/promtool/unittest.go b/cmd/promtool/unittest.go
index 5451c5296c..7030635d1c 100644
--- a/cmd/promtool/unittest.go
+++ b/cmd/promtool/unittest.go
@@ -18,6 +18,7 @@ import (
"encoding/json"
"errors"
"fmt"
+ "io"
"os"
"path/filepath"
"sort"
@@ -29,9 +30,10 @@ import (
"github.com/google/go-cmp/cmp"
"github.com/grafana/regexp"
"github.com/nsf/jsondiff"
- "github.com/prometheus/common/model"
"gopkg.in/yaml.v2"
+ "github.com/prometheus/common/model"
+
"github.com/prometheus/prometheus/model/histogram"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/promql"
@@ -39,12 +41,18 @@ import (
"github.com/prometheus/prometheus/promql/promqltest"
"github.com/prometheus/prometheus/rules"
"github.com/prometheus/prometheus/storage"
+ "github.com/prometheus/prometheus/util/junitxml"
)
// RulesUnitTest does unit testing of rules based on the unit testing files provided.
// More info about the file format can be found in the docs.
func RulesUnitTest(queryOpts promqltest.LazyLoaderOpts, runStrings []string, diffFlag bool, files ...string) int {
+ return RulesUnitTestResult(io.Discard, queryOpts, runStrings, diffFlag, files...)
+}
+
+func RulesUnitTestResult(results io.Writer, queryOpts promqltest.LazyLoaderOpts, runStrings []string, diffFlag bool, files ...string) int {
failed := false
+ junit := &junitxml.JUnitXML{}
var run *regexp.Regexp
if runStrings != nil {
@@ -52,7 +60,7 @@ func RulesUnitTest(queryOpts promqltest.LazyLoaderOpts, runStrings []string, dif
}
for _, f := range files {
- if errs := ruleUnitTest(f, queryOpts, run, diffFlag); errs != nil {
+ if errs := ruleUnitTest(f, queryOpts, run, diffFlag, junit.Suite(f)); errs != nil {
fmt.Fprintln(os.Stderr, " FAILED:")
for _, e := range errs {
fmt.Fprintln(os.Stderr, e.Error())
@@ -64,25 +72,30 @@ func RulesUnitTest(queryOpts promqltest.LazyLoaderOpts, runStrings []string, dif
}
fmt.Println()
}
+ err := junit.WriteXML(results)
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "failed to write JUnit XML: %s\n", err)
+ }
if failed {
return failureExitCode
}
return successExitCode
}
-func ruleUnitTest(filename string, queryOpts promqltest.LazyLoaderOpts, run *regexp.Regexp, diffFlag bool) []error {
- fmt.Println("Unit Testing: ", filename)
-
+func ruleUnitTest(filename string, queryOpts promqltest.LazyLoaderOpts, run *regexp.Regexp, diffFlag bool, ts *junitxml.TestSuite) []error {
b, err := os.ReadFile(filename)
if err != nil {
+ ts.Abort(err)
return []error{err}
}
var unitTestInp unitTestFile
if err := yaml.UnmarshalStrict(b, &unitTestInp); err != nil {
+ ts.Abort(err)
return []error{err}
}
if err := resolveAndGlobFilepaths(filepath.Dir(filename), &unitTestInp); err != nil {
+ ts.Abort(err)
return []error{err}
}
@@ -91,29 +104,38 @@ func ruleUnitTest(filename string, queryOpts promqltest.LazyLoaderOpts, run *reg
}
evalInterval := time.Duration(unitTestInp.EvaluationInterval)
-
+ ts.Settime(time.Now().Format("2006-01-02T15:04:05"))
// Giving number for groups mentioned in the file for ordering.
// Lower number group should be evaluated before higher number group.
groupOrderMap := make(map[string]int)
for i, gn := range unitTestInp.GroupEvalOrder {
if _, ok := groupOrderMap[gn]; ok {
- return []error{fmt.Errorf("group name repeated in evaluation order: %s", gn)}
+ err := fmt.Errorf("group name repeated in evaluation order: %s", gn)
+ ts.Abort(err)
+ return []error{err}
}
groupOrderMap[gn] = i
}
// Testing.
var errs []error
- for _, t := range unitTestInp.Tests {
+ for i, t := range unitTestInp.Tests {
if !matchesRun(t.TestGroupName, run) {
continue
}
-
+ testname := t.TestGroupName
+ if testname == "" {
+ testname = fmt.Sprintf("unnamed#%d", i)
+ }
+ tc := ts.Case(testname)
if t.Interval == 0 {
t.Interval = unitTestInp.EvaluationInterval
}
ers := t.test(evalInterval, groupOrderMap, queryOpts, diffFlag, unitTestInp.RuleFiles...)
if ers != nil {
+ for _, e := range ers {
+ tc.Fail(e.Error())
+ }
errs = append(errs, ers...)
}
}
diff --git a/cmd/promtool/unittest_test.go b/cmd/promtool/unittest_test.go
index 2dbd5a4e51..9bbac28e9f 100644
--- a/cmd/promtool/unittest_test.go
+++ b/cmd/promtool/unittest_test.go
@@ -14,11 +14,15 @@
package main
import (
+ "bytes"
+ "encoding/xml"
+ "fmt"
"testing"
"github.com/stretchr/testify/require"
"github.com/prometheus/prometheus/promql/promqltest"
+ "github.com/prometheus/prometheus/util/junitxml"
)
func TestRulesUnitTest(t *testing.T) {
@@ -125,13 +129,59 @@ func TestRulesUnitTest(t *testing.T) {
want: 0,
},
}
+ reuseFiles := []string{}
+ reuseCount := [2]int{}
for _, tt := range tests {
+ if (tt.queryOpts == promqltest.LazyLoaderOpts{
+ EnableNegativeOffset: true,
+ } || tt.queryOpts == promqltest.LazyLoaderOpts{
+ EnableAtModifier: true,
+ }) {
+ reuseFiles = append(reuseFiles, tt.args.files...)
+ reuseCount[tt.want] += len(tt.args.files)
+ }
t.Run(tt.name, func(t *testing.T) {
if got := RulesUnitTest(tt.queryOpts, nil, false, tt.args.files...); got != tt.want {
t.Errorf("RulesUnitTest() = %v, want %v", got, tt.want)
}
})
}
+ t.Run("Junit xml output ", func(t *testing.T) {
+ var buf bytes.Buffer
+ if got := RulesUnitTestResult(&buf, promqltest.LazyLoaderOpts{}, nil, false, reuseFiles...); got != 1 {
+ t.Errorf("RulesUnitTestResults() = %v, want 1", got)
+ }
+ var test junitxml.JUnitXML
+ output := buf.Bytes()
+ err := xml.Unmarshal(output, &test)
+ if err != nil {
+ fmt.Println("error in decoding XML:", err)
+ return
+ }
+ var total int
+ var passes int
+ var failures int
+ var cases int
+ total = len(test.Suites)
+ if total != len(reuseFiles) {
+ t.Errorf("JUnit output had %d testsuite elements; expected %d\n", total, len(reuseFiles))
+ }
+
+ for _, i := range test.Suites {
+ if i.FailureCount == 0 {
+ passes++
+ } else {
+ failures++
+ }
+ cases += len(i.Cases)
+ }
+ if total != passes+failures {
+ t.Errorf("JUnit output mismatch: Total testsuites (%d) does not equal the sum of passes (%d) and failures (%d).", total, passes, failures)
+ }
+ if cases < total {
+ t.Errorf("JUnit output had %d suites without test cases\n", total-cases)
+ }
+ })
}
func TestRulesUnitTestRun(t *testing.T) {
diff --git a/config/config.go b/config/config.go
index 5615789789..8919f5fee0 100644
--- a/config/config.go
+++ b/config/config.go
@@ -37,6 +37,7 @@ import (
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/model/relabel"
"github.com/prometheus/prometheus/storage/remote/azuread"
+ "github.com/prometheus/prometheus/storage/remote/googleiam"
)
var (
@@ -66,6 +67,11 @@ var (
}
)
+const (
+ LegacyValidationConfig = "legacy"
+ UTF8ValidationConfig = "utf8"
+)
+
// Load parses the YAML input s into a Config.
func Load(s string, expandExternalLabels bool, logger log.Logger) (*Config, error) {
cfg := &Config{}
@@ -180,6 +186,7 @@ var (
// DefaultRemoteWriteConfig is the default remote write configuration.
DefaultRemoteWriteConfig = RemoteWriteConfig{
RemoteTimeout: model.Duration(30 * time.Second),
+ ProtobufMessage: RemoteWriteProtoMsgV1,
QueueConfig: DefaultQueueConfig,
MetadataConfig: DefaultMetadataConfig,
HTTPClientConfig: config.DefaultHTTPClientConfig,
@@ -214,6 +221,7 @@ var (
// DefaultRemoteReadConfig is the default remote read configuration.
DefaultRemoteReadConfig = RemoteReadConfig{
RemoteTimeout: model.Duration(1 * time.Minute),
+ ChunkedReadLimit: DefaultChunkedReadLimit,
HTTPClientConfig: config.DefaultHTTPClientConfig,
FilterExternalLabels: true,
}
@@ -226,6 +234,9 @@ var (
DefaultExemplarsConfig = ExemplarsConfig{
MaxExemplars: 100000,
}
+
+ // DefaultOTLPConfig is the default OTLP configuration.
+ DefaultOTLPConfig = OTLPConfig{}
)
// Config is the top-level configuration for Prometheus's config files.
@@ -241,6 +252,7 @@ type Config struct {
RemoteWriteConfigs []*RemoteWriteConfig `yaml:"remote_write,omitempty"`
RemoteReadConfigs []*RemoteReadConfig `yaml:"remote_read,omitempty"`
+ OTLPConfig OTLPConfig `yaml:"otlp,omitempty"`
}
// SetDirectory joins any relative file paths with dir.
@@ -279,7 +291,7 @@ func (c *Config) GetScrapeConfigs() ([]*ScrapeConfig, error) {
jobNames := map[string]string{}
for i, scfg := range c.ScrapeConfigs {
- // We do these checks for library users that would not call Validate in
+ // We do these checks for library users that would not call validate in
// Unmarshal.
if err := scfg.Validate(c.GlobalConfig); err != nil {
return nil, err
@@ -417,6 +429,8 @@ type GlobalConfig struct {
RuleQueryOffset model.Duration `yaml:"rule_query_offset,omitempty"`
// File to which PromQL queries are logged.
QueryLogFile string `yaml:"query_log_file,omitempty"`
+ // File to which scrape failures are logged.
+ ScrapeFailureLogFile string `yaml:"scrape_failure_log_file,omitempty"`
// The labels to add to any timeseries that this Prometheus instance scrapes.
ExternalLabels labels.Labels `yaml:"external_labels,omitempty"`
// An uncompressed response body larger than this many bytes will cause the
@@ -440,6 +454,8 @@ type GlobalConfig struct {
// Keep no more than this many dropped targets per job.
// 0 means no limit.
KeepDroppedTargets uint `yaml:"keep_dropped_targets,omitempty"`
+ // Allow UTF8 Metric and Label Names.
+ MetricNameValidationScheme string `yaml:"metric_name_validation_scheme,omitempty"`
}
// ScrapeProtocol represents supported protocol for scraping metrics.
@@ -465,6 +481,7 @@ var (
PrometheusText0_0_4 ScrapeProtocol = "PrometheusText0.0.4"
OpenMetricsText0_0_1 ScrapeProtocol = "OpenMetricsText0.0.1"
OpenMetricsText1_0_0 ScrapeProtocol = "OpenMetricsText1.0.0"
+ UTF8NamesHeader string = model.EscapingKey + "=" + model.AllowUTF8
ScrapeProtocolsHeaders = map[ScrapeProtocol]string{
PrometheusProto: "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited",
@@ -514,6 +531,7 @@ func validateAcceptScrapeProtocols(sps []ScrapeProtocol) error {
// SetDirectory joins any relative file paths with dir.
func (c *GlobalConfig) SetDirectory(dir string) {
c.QueryLogFile = config.JoinDir(dir, c.QueryLogFile)
+ c.ScrapeFailureLogFile = config.JoinDir(dir, c.ScrapeFailureLogFile)
}
// UnmarshalYAML implements the yaml.Unmarshaler interface.
@@ -576,6 +594,7 @@ func (c *GlobalConfig) isZero() bool {
c.EvaluationInterval == 0 &&
c.RuleQueryOffset == 0 &&
c.QueryLogFile == "" &&
+ c.ScrapeFailureLogFile == "" &&
c.ScrapeProtocols == nil
}
@@ -619,6 +638,8 @@ type ScrapeConfig struct {
ScrapeClassicHistograms bool `yaml:"scrape_classic_histograms,omitempty"`
// Whether to convert a scraped classic histogram into a native histogram with custom buckets.
ConvertClassicHistograms bool `yaml:"convert_classic_histograms,omitempty"`
+ // File to which scrape failures are logged.
+ ScrapeFailureLogFile string `yaml:"scrape_failure_log_file,omitempty"`
// The HTTP resource path on which to fetch metrics from targets.
MetricsPath string `yaml:"metrics_path,omitempty"`
// The URL scheme with which to fetch metrics from targets.
@@ -652,6 +673,8 @@ type ScrapeConfig struct {
// Keep no more than this many dropped targets per job.
// 0 means no limit.
KeepDroppedTargets uint `yaml:"keep_dropped_targets,omitempty"`
+ // Allow UTF8 Metric and Label Names.
+ MetricNameValidationScheme string `yaml:"metric_name_validation_scheme,omitempty"`
// We cannot do proper Go type embedding below as the parser will then parse
// values arbitrarily into the overflow maps of further-down types.
@@ -669,6 +692,7 @@ type ScrapeConfig struct {
func (c *ScrapeConfig) SetDirectory(dir string) {
c.ServiceDiscoveryConfigs.SetDirectory(dir)
c.HTTPClientConfig.SetDirectory(dir)
+ c.ScrapeFailureLogFile = config.JoinDir(dir, c.ScrapeFailureLogFile)
}
// UnmarshalYAML implements the yaml.Unmarshaler interface.
@@ -750,6 +774,9 @@ func (c *ScrapeConfig) Validate(globalConfig GlobalConfig) error {
if c.KeepDroppedTargets == 0 {
c.KeepDroppedTargets = globalConfig.KeepDroppedTargets
}
+ if c.ScrapeFailureLogFile == "" {
+ c.ScrapeFailureLogFile = globalConfig.ScrapeFailureLogFile
+ }
if c.ScrapeProtocols == nil {
c.ScrapeProtocols = globalConfig.ScrapeProtocols
@@ -758,6 +785,19 @@ func (c *ScrapeConfig) Validate(globalConfig GlobalConfig) error {
return fmt.Errorf("%w for scrape config with job name %q", err, c.JobName)
}
+ switch globalConfig.MetricNameValidationScheme {
+ case LegacyValidationConfig:
+ case "", UTF8ValidationConfig:
+ if model.NameValidationScheme != model.UTF8Validation {
+ panic("utf8 name validation requested but model.NameValidationScheme is not set to UTF8")
+ }
+ default:
+ return fmt.Errorf("unknown name validation method specified, must be either 'legacy' or 'utf8', got %s", globalConfig.MetricNameValidationScheme)
+ }
+ if c.MetricNameValidationScheme == "" {
+ c.MetricNameValidationScheme = globalConfig.MetricNameValidationScheme
+ }
+
return nil
}
@@ -1057,6 +1097,50 @@ func CheckTargetAddress(address model.LabelValue) error {
return nil
}
+// RemoteWriteProtoMsg represents the known protobuf message for the remote write
+// 1.0 and 2.0 specs.
+type RemoteWriteProtoMsg string
+
+// Validate returns error if the given reference for the protobuf message is not supported.
+func (s RemoteWriteProtoMsg) Validate() error {
+ switch s {
+ case RemoteWriteProtoMsgV1, RemoteWriteProtoMsgV2:
+ return nil
+ default:
+ return fmt.Errorf("unknown remote write protobuf message %v, supported: %v", s, RemoteWriteProtoMsgs{RemoteWriteProtoMsgV1, RemoteWriteProtoMsgV2}.String())
+ }
+}
+
+type RemoteWriteProtoMsgs []RemoteWriteProtoMsg
+
+func (m RemoteWriteProtoMsgs) Strings() []string {
+ ret := make([]string, 0, len(m))
+ for _, typ := range m {
+ ret = append(ret, string(typ))
+ }
+ return ret
+}
+
+func (m RemoteWriteProtoMsgs) String() string {
+ return strings.Join(m.Strings(), ", ")
+}
+
+var (
+ // RemoteWriteProtoMsgV1 represents the `prometheus.WriteRequest` protobuf
+ // message introduced in the https://prometheus.io/docs/specs/remote_write_spec/,
+ // which will eventually be deprecated.
+ //
+ // NOTE: This string is used for both HTTP header values and config value, so don't change
+ // this reference.
+ RemoteWriteProtoMsgV1 RemoteWriteProtoMsg = "prometheus.WriteRequest"
+ // RemoteWriteProtoMsgV2 represents the `io.prometheus.write.v2.Request` protobuf
+ // message introduced in https://prometheus.io/docs/specs/remote_write_spec_2_0/
+ //
+ // NOTE: This string is used for both HTTP header values and config value, so don't change
+ // this reference.
+ RemoteWriteProtoMsgV2 RemoteWriteProtoMsg = "io.prometheus.write.v2.Request"
+)
+
// RemoteWriteConfig is the configuration for writing to remote storage.
type RemoteWriteConfig struct {
URL *config.URL `yaml:"url"`
@@ -1066,6 +1150,9 @@ type RemoteWriteConfig struct {
Name string `yaml:"name,omitempty"`
SendExemplars bool `yaml:"send_exemplars,omitempty"`
SendNativeHistograms bool `yaml:"send_native_histograms,omitempty"`
+ // ProtobufMessage specifies the protobuf message to use against the remote
+ // receiver as specified in https://prometheus.io/docs/specs/remote_write_spec_2_0/
+ ProtobufMessage RemoteWriteProtoMsg `yaml:"protobuf_message,omitempty"`
// We cannot do proper Go type embedding below as the parser will then parse
// values arbitrarily into the overflow maps of further-down types.
@@ -1074,6 +1161,7 @@ type RemoteWriteConfig struct {
MetadataConfig MetadataConfig `yaml:"metadata_config,omitempty"`
SigV4Config *sigv4.SigV4Config `yaml:"sigv4,omitempty"`
AzureADConfig *azuread.AzureADConfig `yaml:"azuread,omitempty"`
+ GoogleIAMConfig *googleiam.Config `yaml:"google_iam,omitempty"`
}
// SetDirectory joins any relative file paths with dir.
@@ -1100,6 +1188,10 @@ func (c *RemoteWriteConfig) UnmarshalYAML(unmarshal func(interface{}) error) err
return err
}
+ if err := c.ProtobufMessage.Validate(); err != nil {
+ return fmt.Errorf("invalid protobuf_message value: %w", err)
+ }
+
// The UnmarshalYAML method of HTTPClientConfig is not being called because it's not a pointer.
// We cannot make it a pointer as the parser panics for inlined pointer structs.
// Thus we just do its validation here.
@@ -1107,17 +1199,33 @@ func (c *RemoteWriteConfig) UnmarshalYAML(unmarshal func(interface{}) error) err
return err
}
- httpClientConfigAuthEnabled := c.HTTPClientConfig.BasicAuth != nil ||
- c.HTTPClientConfig.Authorization != nil || c.HTTPClientConfig.OAuth2 != nil
+ return validateAuthConfigs(c)
+}
- if httpClientConfigAuthEnabled && (c.SigV4Config != nil || c.AzureADConfig != nil) {
- return fmt.Errorf("at most one of basic_auth, authorization, oauth2, sigv4, & azuread must be configured")
+// validateAuthConfigs validates that at most one of basic_auth, authorization, oauth2, sigv4, azuread or google_iam must be configured.
+func validateAuthConfigs(c *RemoteWriteConfig) error {
+ var authConfigured []string
+ if c.HTTPClientConfig.BasicAuth != nil {
+ authConfigured = append(authConfigured, "basic_auth")
}
-
- if c.SigV4Config != nil && c.AzureADConfig != nil {
- return fmt.Errorf("at most one of basic_auth, authorization, oauth2, sigv4, & azuread must be configured")
+ if c.HTTPClientConfig.Authorization != nil {
+ authConfigured = append(authConfigured, "authorization")
+ }
+ if c.HTTPClientConfig.OAuth2 != nil {
+ authConfigured = append(authConfigured, "oauth2")
+ }
+ if c.SigV4Config != nil {
+ authConfigured = append(authConfigured, "sigv4")
+ }
+ if c.AzureADConfig != nil {
+ authConfigured = append(authConfigured, "azuread")
+ }
+ if c.GoogleIAMConfig != nil {
+ authConfigured = append(authConfigured, "google_iam")
+ }
+ if len(authConfigured) > 1 {
+ return fmt.Errorf("at most one of basic_auth, authorization, oauth2, sigv4, azuread or google_iam must be configured. Currently configured: %v", authConfigured)
}
-
return nil
}
@@ -1136,7 +1244,7 @@ func validateHeadersForTracing(headers map[string]string) error {
func validateHeaders(headers map[string]string) error {
for header := range headers {
if strings.ToLower(header) == "authorization" {
- return errors.New("authorization header must be changed via the basic_auth, authorization, oauth2, sigv4, or azuread parameter")
+ return errors.New("authorization header must be changed via the basic_auth, authorization, oauth2, sigv4, azuread or google_iam parameter")
}
if _, ok := reservedHeaders[strings.ToLower(header)]; ok {
return fmt.Errorf("%s is a reserved header. It must not be changed", header)
@@ -1184,13 +1292,20 @@ type MetadataConfig struct {
MaxSamplesPerSend int `yaml:"max_samples_per_send,omitempty"`
}
+const (
+ // DefaultChunkedReadLimit is the default value for the maximum size of the protobuf frame client allows.
+ // 50MB is the default. This is equivalent to ~100k full XOR chunks and average labelset.
+ DefaultChunkedReadLimit = 5e+7
+)
+
// RemoteReadConfig is the configuration for reading from remote storage.
type RemoteReadConfig struct {
- URL *config.URL `yaml:"url"`
- RemoteTimeout model.Duration `yaml:"remote_timeout,omitempty"`
- Headers map[string]string `yaml:"headers,omitempty"`
- ReadRecent bool `yaml:"read_recent,omitempty"`
- Name string `yaml:"name,omitempty"`
+ URL *config.URL `yaml:"url"`
+ RemoteTimeout model.Duration `yaml:"remote_timeout,omitempty"`
+ ChunkedReadLimit uint64 `yaml:"chunked_read_limit,omitempty"`
+ Headers map[string]string `yaml:"headers,omitempty"`
+ ReadRecent bool `yaml:"read_recent,omitempty"`
+ Name string `yaml:"name,omitempty"`
// We cannot do proper Go type embedding below as the parser will then parse
// values arbitrarily into the overflow maps of further-down types.
@@ -1255,3 +1370,35 @@ func getGoGCEnv() int {
}
return DefaultRuntimeConfig.GoGC
}
+
+// OTLPConfig is the configuration for writing to the OTLP endpoint.
+type OTLPConfig struct {
+ PromoteResourceAttributes []string `yaml:"promote_resource_attributes,omitempty"`
+}
+
+// UnmarshalYAML implements the yaml.Unmarshaler interface.
+func (c *OTLPConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
+ *c = DefaultOTLPConfig
+ type plain OTLPConfig
+ if err := unmarshal((*plain)(c)); err != nil {
+ return err
+ }
+
+ seen := map[string]struct{}{}
+ var err error
+ for i, attr := range c.PromoteResourceAttributes {
+ attr = strings.TrimSpace(attr)
+ if attr == "" {
+ err = errors.Join(err, fmt.Errorf("empty promoted OTel resource attribute"))
+ continue
+ }
+ if _, exists := seen[attr]; exists {
+ err = errors.Join(err, fmt.Errorf("duplicated promoted OTel resource attribute %q", attr))
+ continue
+ }
+
+ seen[attr] = struct{}{}
+ c.PromoteResourceAttributes[i] = attr
+ }
+ return err
+}
diff --git a/config/config_test.go b/config/config_test.go
index d84059b48f..66377f6879 100644
--- a/config/config_test.go
+++ b/config/config_test.go
@@ -16,6 +16,7 @@ package config
import (
"crypto/tls"
"encoding/json"
+ "fmt"
"net/url"
"os"
"path/filepath"
@@ -61,6 +62,11 @@ import (
"github.com/prometheus/prometheus/util/testutil"
)
+func init() {
+ // This can be removed when the default validation scheme in common is updated.
+ model.NameValidationScheme = model.UTF8Validation
+}
+
func mustParseURL(u string) *config.URL {
parsed, err := url.Parse(u)
if err != nil {
@@ -77,14 +83,16 @@ const (
globLabelNameLengthLimit = 200
globLabelValueLengthLimit = 200
globalGoGC = 42
+ globScrapeFailureLogFile = "testdata/fail.log"
)
var expectedConf = &Config{
GlobalConfig: GlobalConfig{
- ScrapeInterval: model.Duration(15 * time.Second),
- ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout,
- EvaluationInterval: model.Duration(30 * time.Second),
- QueryLogFile: "",
+ ScrapeInterval: model.Duration(15 * time.Second),
+ ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout,
+ EvaluationInterval: model.Duration(30 * time.Second),
+ QueryLogFile: "testdata/query.log",
+ ScrapeFailureLogFile: globScrapeFailureLogFile,
ExternalLabels: labels.FromStrings("foo", "bar", "monitor", "codelab"),
@@ -108,9 +116,10 @@ var expectedConf = &Config{
RemoteWriteConfigs: []*RemoteWriteConfig{
{
- URL: mustParseURL("http://remote1/push"),
- RemoteTimeout: model.Duration(30 * time.Second),
- Name: "drop_expensive",
+ URL: mustParseURL("http://remote1/push"),
+ ProtobufMessage: RemoteWriteProtoMsgV1,
+ RemoteTimeout: model.Duration(30 * time.Second),
+ Name: "drop_expensive",
WriteRelabelConfigs: []*relabel.Config{
{
SourceLabels: model.LabelNames{"__name__"},
@@ -137,11 +146,12 @@ var expectedConf = &Config{
},
},
{
- URL: mustParseURL("http://remote2/push"),
- RemoteTimeout: model.Duration(30 * time.Second),
- QueueConfig: DefaultQueueConfig,
- MetadataConfig: DefaultMetadataConfig,
- Name: "rw_tls",
+ URL: mustParseURL("http://remote2/push"),
+ ProtobufMessage: RemoteWriteProtoMsgV2,
+ RemoteTimeout: model.Duration(30 * time.Second),
+ QueueConfig: DefaultQueueConfig,
+ MetadataConfig: DefaultMetadataConfig,
+ Name: "rw_tls",
HTTPClientConfig: config.HTTPClientConfig{
TLSConfig: config.TLSConfig{
CertFile: filepath.FromSlash("testdata/valid_cert_file"),
@@ -154,12 +164,19 @@ var expectedConf = &Config{
},
},
+ OTLPConfig: OTLPConfig{
+ PromoteResourceAttributes: []string{
+ "k8s.cluster.name", "k8s.job.name", "k8s.namespace.name",
+ },
+ },
+
RemoteReadConfigs: []*RemoteReadConfig{
{
- URL: mustParseURL("http://remote1/read"),
- RemoteTimeout: model.Duration(1 * time.Minute),
- ReadRecent: true,
- Name: "default",
+ URL: mustParseURL("http://remote1/read"),
+ RemoteTimeout: model.Duration(1 * time.Minute),
+ ChunkedReadLimit: DefaultChunkedReadLimit,
+ ReadRecent: true,
+ Name: "default",
HTTPClientConfig: config.HTTPClientConfig{
FollowRedirects: true,
EnableHTTP2: false,
@@ -169,6 +186,7 @@ var expectedConf = &Config{
{
URL: mustParseURL("http://remote3/read"),
RemoteTimeout: model.Duration(1 * time.Minute),
+ ChunkedReadLimit: DefaultChunkedReadLimit,
ReadRecent: false,
Name: "read_special",
RequiredMatchers: model.LabelSet{"job": "special"},
@@ -200,6 +218,7 @@ var expectedConf = &Config{
LabelNameLengthLimit: globLabelNameLengthLimit,
LabelValueLengthLimit: globLabelValueLengthLimit,
ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols,
+ ScrapeFailureLogFile: "testdata/fail_prom.log",
MetricsPath: DefaultScrapeConfig.MetricsPath,
Scheme: DefaultScrapeConfig.Scheme,
@@ -214,6 +233,15 @@ var expectedConf = &Config{
TLSConfig: config.TLSConfig{
MinVersion: config.TLSVersion(tls.VersionTLS10),
},
+ HTTPHeaders: &config.Headers{
+ Headers: map[string]config.Header{
+ "foo": {
+ Values: []string{"foobar"},
+ Secrets: []config.Secret{"bar", "foo"},
+ Files: []string{filepath.FromSlash("testdata/valid_password_file")},
+ },
+ },
+ },
},
ServiceDiscoveryConfigs: discovery.Configs{
@@ -303,6 +331,7 @@ var expectedConf = &Config{
LabelNameLengthLimit: 210,
LabelValueLengthLimit: 210,
ScrapeProtocols: []ScrapeProtocol{PrometheusText0_0_4},
+ ScrapeFailureLogFile: globScrapeFailureLogFile,
HTTPClientConfig: config.HTTPClientConfig{
BasicAuth: &config.BasicAuth{
@@ -400,6 +429,7 @@ var expectedConf = &Config{
LabelNameLengthLimit: globLabelNameLengthLimit,
LabelValueLengthLimit: globLabelValueLengthLimit,
ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols,
+ ScrapeFailureLogFile: globScrapeFailureLogFile,
MetricsPath: DefaultScrapeConfig.MetricsPath,
Scheme: DefaultScrapeConfig.Scheme,
@@ -455,6 +485,7 @@ var expectedConf = &Config{
LabelNameLengthLimit: globLabelNameLengthLimit,
LabelValueLengthLimit: globLabelValueLengthLimit,
ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols,
+ ScrapeFailureLogFile: globScrapeFailureLogFile,
MetricsPath: "/metrics",
Scheme: "http",
@@ -488,6 +519,7 @@ var expectedConf = &Config{
LabelNameLengthLimit: globLabelNameLengthLimit,
LabelValueLengthLimit: globLabelValueLengthLimit,
ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols,
+ ScrapeFailureLogFile: globScrapeFailureLogFile,
MetricsPath: DefaultScrapeConfig.MetricsPath,
Scheme: DefaultScrapeConfig.Scheme,
@@ -527,6 +559,7 @@ var expectedConf = &Config{
LabelNameLengthLimit: globLabelNameLengthLimit,
LabelValueLengthLimit: globLabelValueLengthLimit,
ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols,
+ ScrapeFailureLogFile: globScrapeFailureLogFile,
MetricsPath: DefaultScrapeConfig.MetricsPath,
Scheme: DefaultScrapeConfig.Scheme,
@@ -566,6 +599,7 @@ var expectedConf = &Config{
LabelNameLengthLimit: globLabelNameLengthLimit,
LabelValueLengthLimit: globLabelValueLengthLimit,
ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols,
+ ScrapeFailureLogFile: globScrapeFailureLogFile,
MetricsPath: DefaultScrapeConfig.MetricsPath,
Scheme: DefaultScrapeConfig.Scheme,
@@ -595,6 +629,7 @@ var expectedConf = &Config{
LabelNameLengthLimit: globLabelNameLengthLimit,
LabelValueLengthLimit: globLabelValueLengthLimit,
ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols,
+ ScrapeFailureLogFile: globScrapeFailureLogFile,
MetricsPath: DefaultScrapeConfig.MetricsPath,
Scheme: DefaultScrapeConfig.Scheme,
@@ -632,6 +667,7 @@ var expectedConf = &Config{
LabelNameLengthLimit: globLabelNameLengthLimit,
LabelValueLengthLimit: globLabelValueLengthLimit,
ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols,
+ ScrapeFailureLogFile: globScrapeFailureLogFile,
MetricsPath: DefaultScrapeConfig.MetricsPath,
Scheme: DefaultScrapeConfig.Scheme,
@@ -666,6 +702,7 @@ var expectedConf = &Config{
LabelNameLengthLimit: globLabelNameLengthLimit,
LabelValueLengthLimit: globLabelValueLengthLimit,
ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols,
+ ScrapeFailureLogFile: globScrapeFailureLogFile,
MetricsPath: DefaultScrapeConfig.MetricsPath,
Scheme: DefaultScrapeConfig.Scheme,
@@ -707,6 +744,7 @@ var expectedConf = &Config{
LabelNameLengthLimit: globLabelNameLengthLimit,
LabelValueLengthLimit: globLabelValueLengthLimit,
ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols,
+ ScrapeFailureLogFile: globScrapeFailureLogFile,
MetricsPath: DefaultScrapeConfig.MetricsPath,
Scheme: DefaultScrapeConfig.Scheme,
@@ -738,6 +776,7 @@ var expectedConf = &Config{
LabelNameLengthLimit: globLabelNameLengthLimit,
LabelValueLengthLimit: globLabelValueLengthLimit,
ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols,
+ ScrapeFailureLogFile: globScrapeFailureLogFile,
MetricsPath: DefaultScrapeConfig.MetricsPath,
Scheme: DefaultScrapeConfig.Scheme,
@@ -772,6 +811,7 @@ var expectedConf = &Config{
LabelNameLengthLimit: globLabelNameLengthLimit,
LabelValueLengthLimit: globLabelValueLengthLimit,
ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols,
+ ScrapeFailureLogFile: globScrapeFailureLogFile,
MetricsPath: DefaultScrapeConfig.MetricsPath,
Scheme: DefaultScrapeConfig.Scheme,
@@ -799,6 +839,7 @@ var expectedConf = &Config{
LabelNameLengthLimit: globLabelNameLengthLimit,
LabelValueLengthLimit: globLabelValueLengthLimit,
ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols,
+ ScrapeFailureLogFile: globScrapeFailureLogFile,
MetricsPath: DefaultScrapeConfig.MetricsPath,
Scheme: DefaultScrapeConfig.Scheme,
@@ -829,6 +870,7 @@ var expectedConf = &Config{
LabelNameLengthLimit: globLabelNameLengthLimit,
LabelValueLengthLimit: globLabelValueLengthLimit,
ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols,
+ ScrapeFailureLogFile: globScrapeFailureLogFile,
MetricsPath: "/federate",
Scheme: DefaultScrapeConfig.Scheme,
@@ -859,6 +901,7 @@ var expectedConf = &Config{
LabelNameLengthLimit: globLabelNameLengthLimit,
LabelValueLengthLimit: globLabelValueLengthLimit,
ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols,
+ ScrapeFailureLogFile: globScrapeFailureLogFile,
MetricsPath: DefaultScrapeConfig.MetricsPath,
Scheme: DefaultScrapeConfig.Scheme,
@@ -889,6 +932,7 @@ var expectedConf = &Config{
LabelNameLengthLimit: globLabelNameLengthLimit,
LabelValueLengthLimit: globLabelValueLengthLimit,
ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols,
+ ScrapeFailureLogFile: globScrapeFailureLogFile,
MetricsPath: DefaultScrapeConfig.MetricsPath,
Scheme: DefaultScrapeConfig.Scheme,
@@ -916,6 +960,7 @@ var expectedConf = &Config{
LabelNameLengthLimit: globLabelNameLengthLimit,
LabelValueLengthLimit: globLabelValueLengthLimit,
ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols,
+ ScrapeFailureLogFile: globScrapeFailureLogFile,
MetricsPath: DefaultScrapeConfig.MetricsPath,
Scheme: DefaultScrapeConfig.Scheme,
@@ -951,6 +996,7 @@ var expectedConf = &Config{
LabelNameLengthLimit: globLabelNameLengthLimit,
LabelValueLengthLimit: globLabelValueLengthLimit,
ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols,
+ ScrapeFailureLogFile: globScrapeFailureLogFile,
MetricsPath: DefaultScrapeConfig.MetricsPath,
Scheme: DefaultScrapeConfig.Scheme,
@@ -985,6 +1031,7 @@ var expectedConf = &Config{
LabelNameLengthLimit: globLabelNameLengthLimit,
LabelValueLengthLimit: globLabelValueLengthLimit,
ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols,
+ ScrapeFailureLogFile: globScrapeFailureLogFile,
MetricsPath: DefaultScrapeConfig.MetricsPath,
Scheme: DefaultScrapeConfig.Scheme,
@@ -1016,6 +1063,7 @@ var expectedConf = &Config{
LabelNameLengthLimit: globLabelNameLengthLimit,
LabelValueLengthLimit: globLabelValueLengthLimit,
ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols,
+ ScrapeFailureLogFile: globScrapeFailureLogFile,
MetricsPath: DefaultScrapeConfig.MetricsPath,
Scheme: DefaultScrapeConfig.Scheme,
@@ -1046,6 +1094,7 @@ var expectedConf = &Config{
LabelNameLengthLimit: globLabelNameLengthLimit,
LabelValueLengthLimit: globLabelValueLengthLimit,
ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols,
+ ScrapeFailureLogFile: globScrapeFailureLogFile,
MetricsPath: DefaultScrapeConfig.MetricsPath,
Scheme: DefaultScrapeConfig.Scheme,
@@ -1080,6 +1129,7 @@ var expectedConf = &Config{
LabelNameLengthLimit: globLabelNameLengthLimit,
LabelValueLengthLimit: globLabelValueLengthLimit,
ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols,
+ ScrapeFailureLogFile: globScrapeFailureLogFile,
MetricsPath: DefaultScrapeConfig.MetricsPath,
Scheme: DefaultScrapeConfig.Scheme,
@@ -1117,6 +1167,7 @@ var expectedConf = &Config{
LabelNameLengthLimit: globLabelNameLengthLimit,
LabelValueLengthLimit: globLabelValueLengthLimit,
ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols,
+ ScrapeFailureLogFile: globScrapeFailureLogFile,
MetricsPath: DefaultScrapeConfig.MetricsPath,
Scheme: DefaultScrapeConfig.Scheme,
@@ -1173,6 +1224,7 @@ var expectedConf = &Config{
LabelNameLengthLimit: globLabelNameLengthLimit,
LabelValueLengthLimit: globLabelValueLengthLimit,
ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols,
+ ScrapeFailureLogFile: globScrapeFailureLogFile,
MetricsPath: DefaultScrapeConfig.MetricsPath,
Scheme: DefaultScrapeConfig.Scheme,
@@ -1200,6 +1252,7 @@ var expectedConf = &Config{
LabelNameLengthLimit: globLabelNameLengthLimit,
LabelValueLengthLimit: globLabelValueLengthLimit,
ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols,
+ ScrapeFailureLogFile: globScrapeFailureLogFile,
HTTPClientConfig: config.DefaultHTTPClientConfig,
MetricsPath: DefaultScrapeConfig.MetricsPath,
@@ -1238,6 +1291,7 @@ var expectedConf = &Config{
LabelNameLengthLimit: globLabelNameLengthLimit,
LabelValueLengthLimit: globLabelValueLengthLimit,
ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols,
+ ScrapeFailureLogFile: globScrapeFailureLogFile,
HTTPClientConfig: config.DefaultHTTPClientConfig,
MetricsPath: DefaultScrapeConfig.MetricsPath,
@@ -1282,6 +1336,7 @@ var expectedConf = &Config{
LabelNameLengthLimit: globLabelNameLengthLimit,
LabelValueLengthLimit: globLabelValueLengthLimit,
ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols,
+ ScrapeFailureLogFile: globScrapeFailureLogFile,
MetricsPath: DefaultScrapeConfig.MetricsPath,
Scheme: DefaultScrapeConfig.Scheme,
@@ -1317,6 +1372,7 @@ var expectedConf = &Config{
LabelNameLengthLimit: globLabelNameLengthLimit,
LabelValueLengthLimit: globLabelValueLengthLimit,
ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols,
+ ScrapeFailureLogFile: globScrapeFailureLogFile,
HTTPClientConfig: config.DefaultHTTPClientConfig,
MetricsPath: DefaultScrapeConfig.MetricsPath,
@@ -1346,6 +1402,7 @@ var expectedConf = &Config{
LabelNameLengthLimit: globLabelNameLengthLimit,
LabelValueLengthLimit: globLabelValueLengthLimit,
ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols,
+ ScrapeFailureLogFile: globScrapeFailureLogFile,
MetricsPath: DefaultScrapeConfig.MetricsPath,
Scheme: DefaultScrapeConfig.Scheme,
@@ -1378,6 +1435,7 @@ var expectedConf = &Config{
LabelNameLengthLimit: globLabelNameLengthLimit,
LabelValueLengthLimit: globLabelValueLengthLimit,
ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols,
+ ScrapeFailureLogFile: globScrapeFailureLogFile,
MetricsPath: DefaultScrapeConfig.MetricsPath,
Scheme: DefaultScrapeConfig.Scheme,
@@ -1469,6 +1527,26 @@ func TestRemoteWriteRetryOnRateLimit(t *testing.T) {
require.False(t, got.RemoteWriteConfigs[1].QueueConfig.RetryOnRateLimit)
}
+func TestOTLPSanitizeResourceAttributes(t *testing.T) {
+ t.Run("good config", func(t *testing.T) {
+ want, err := LoadFile(filepath.Join("testdata", "otlp_sanitize_resource_attributes.good.yml"), false, false, log.NewNopLogger())
+ require.NoError(t, err)
+
+ out, err := yaml.Marshal(want)
+ require.NoError(t, err)
+ var got Config
+ require.NoError(t, yaml.UnmarshalStrict(out, &got))
+
+ require.Equal(t, []string{"k8s.cluster.name", "k8s.job.name", "k8s.namespace.name"}, got.OTLPConfig.PromoteResourceAttributes)
+ })
+
+ t.Run("bad config", func(t *testing.T) {
+ _, err := LoadFile(filepath.Join("testdata", "otlp_sanitize_resource_attributes.bad.yml"), false, false, log.NewNopLogger())
+ require.ErrorContains(t, err, `duplicated promoted OTel resource attribute "k8s.job.name"`)
+ require.ErrorContains(t, err, `empty promoted OTel resource attribute`)
+ })
+}
+
func TestLoadConfig(t *testing.T) {
// Parse a valid file that sets a global scrape timeout. This tests whether parsing
// an overwritten default field in the global config permanently changes the default.
@@ -1501,7 +1579,7 @@ func TestElideSecrets(t *testing.T) {
yamlConfig := string(config)
matches := secretRe.FindAllStringIndex(yamlConfig, -1)
- require.Len(t, matches, 22, "wrong number of secret matches found")
+ require.Len(t, matches, 24, "wrong number of secret matches found")
require.NotContains(t, yamlConfig, "mysecret",
"yaml marshal reveals authentication credentials.")
}
@@ -1798,7 +1876,11 @@ var expectedErrors = []struct {
},
{
filename: "remote_write_authorization_header.bad.yml",
- errMsg: `authorization header must be changed via the basic_auth, authorization, oauth2, sigv4, or azuread parameter`,
+ errMsg: `authorization header must be changed via the basic_auth, authorization, oauth2, sigv4, azuread or google_iam parameter`,
+ },
+ {
+ filename: "remote_write_wrong_msg.bad.yml",
+ errMsg: `invalid protobuf_message value: unknown remote write protobuf message io.prometheus.writet.v2.Request, supported: prometheus.WriteRequest, io.prometheus.write.v2.Request`,
},
{
filename: "remote_write_url_missing.bad.yml",
@@ -2007,6 +2089,10 @@ var expectedErrors = []struct {
}
func TestBadConfigs(t *testing.T) {
+ model.NameValidationScheme = model.LegacyValidation
+ defer func() {
+ model.NameValidationScheme = model.UTF8Validation
+ }()
for _, ee := range expectedErrors {
_, err := LoadFile("testdata/"+ee.filename, false, false, log.NewNopLogger())
require.Error(t, err, "%s", ee.filename)
@@ -2016,6 +2102,10 @@ func TestBadConfigs(t *testing.T) {
}
func TestBadStaticConfigsJSON(t *testing.T) {
+ model.NameValidationScheme = model.LegacyValidation
+ defer func() {
+ model.NameValidationScheme = model.UTF8Validation
+ }()
content, err := os.ReadFile("testdata/static_config.bad.json")
require.NoError(t, err)
var tg targetgroup.Group
@@ -2024,6 +2114,10 @@ func TestBadStaticConfigsJSON(t *testing.T) {
}
func TestBadStaticConfigsYML(t *testing.T) {
+ model.NameValidationScheme = model.LegacyValidation
+ defer func() {
+ model.NameValidationScheme = model.UTF8Validation
+ }()
content, err := os.ReadFile("testdata/static_config.bad.yml")
require.NoError(t, err)
var tg targetgroup.Group
@@ -2268,3 +2362,52 @@ func TestScrapeConfigDisableCompression(t *testing.T) {
require.False(t, got.ScrapeConfigs[0].EnableCompression)
}
+
+func TestScrapeConfigNameValidationSettings(t *testing.T) {
+ model.NameValidationScheme = model.UTF8Validation
+ defer func() {
+ model.NameValidationScheme = model.LegacyValidation
+ }()
+
+ tests := []struct {
+ name string
+ inputFile string
+ expectScheme string
+ }{
+ {
+ name: "blank config implies default",
+ inputFile: "scrape_config_default_validation_mode",
+ expectScheme: "",
+ },
+ {
+ name: "global setting implies local settings",
+ inputFile: "scrape_config_global_validation_mode",
+ expectScheme: "legacy",
+ },
+ {
+ name: "local setting",
+ inputFile: "scrape_config_local_validation_mode",
+ expectScheme: "legacy",
+ },
+ {
+ name: "local setting overrides global setting",
+ inputFile: "scrape_config_local_global_validation_mode",
+ expectScheme: "utf8",
+ },
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ want, err := LoadFile(fmt.Sprintf("testdata/%s.yml", tc.inputFile), false, false, log.NewNopLogger())
+ require.NoError(t, err)
+
+ out, err := yaml.Marshal(want)
+
+ require.NoError(t, err)
+ got := &Config{}
+ require.NoError(t, yaml.UnmarshalStrict(out, got))
+
+ require.Equal(t, tc.expectScheme, got.ScrapeConfigs[0].MetricNameValidationScheme)
+ })
+ }
+}
diff --git a/config/reload.go b/config/reload.go
new file mode 100644
index 0000000000..8be1b28d8a
--- /dev/null
+++ b/config/reload.go
@@ -0,0 +1,92 @@
+// Copyright 2024 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+import (
+ "crypto/sha256"
+ "encoding/hex"
+ "fmt"
+ "os"
+ "path/filepath"
+
+ "gopkg.in/yaml.v2"
+)
+
+type ExternalFilesConfig struct {
+ RuleFiles []string `yaml:"rule_files"`
+ ScrapeConfigFiles []string `yaml:"scrape_config_files"`
+}
+
+// GenerateChecksum generates a checksum of the YAML file and the files it references.
+func GenerateChecksum(yamlFilePath string) (string, error) {
+ hash := sha256.New()
+
+ yamlContent, err := os.ReadFile(yamlFilePath)
+ if err != nil {
+ return "", fmt.Errorf("error reading YAML file: %w", err)
+ }
+ _, err = hash.Write(yamlContent)
+ if err != nil {
+ return "", fmt.Errorf("error writing YAML file to hash: %w", err)
+ }
+
+ var config ExternalFilesConfig
+ if err := yaml.Unmarshal(yamlContent, &config); err != nil {
+ return "", fmt.Errorf("error unmarshalling YAML: %w", err)
+ }
+
+ dir := filepath.Dir(yamlFilePath)
+
+ for i, file := range config.RuleFiles {
+ config.RuleFiles[i] = filepath.Join(dir, file)
+ }
+ for i, file := range config.ScrapeConfigFiles {
+ config.ScrapeConfigFiles[i] = filepath.Join(dir, file)
+ }
+
+ files := map[string][]string{
+ "r": config.RuleFiles, // "r" for rule files
+ "s": config.ScrapeConfigFiles, // "s" for scrape config files
+ }
+
+ for _, prefix := range []string{"r", "s"} {
+ for _, pattern := range files[prefix] {
+ matchingFiles, err := filepath.Glob(pattern)
+ if err != nil {
+ return "", fmt.Errorf("error finding files with pattern %q: %w", pattern, err)
+ }
+
+ for _, file := range matchingFiles {
+ // Write prefix to the hash ("r" or "s") followed by \0, then
+ // the file path.
+ _, err = hash.Write([]byte(prefix + "\x00" + file + "\x00"))
+ if err != nil {
+ return "", fmt.Errorf("error writing %q path to hash: %w", file, err)
+ }
+
+ // Read and hash the content of the file.
+ content, err := os.ReadFile(file)
+ if err != nil {
+ return "", fmt.Errorf("error reading file %s: %w", file, err)
+ }
+ _, err = hash.Write(append(content, []byte("\x00")...))
+ if err != nil {
+ return "", fmt.Errorf("error writing %q content to hash: %w", file, err)
+ }
+ }
+ }
+ }
+
+ return hex.EncodeToString(hash.Sum(nil)), nil
+}
diff --git a/config/reload_test.go b/config/reload_test.go
new file mode 100644
index 0000000000..f0f44f3588
--- /dev/null
+++ b/config/reload_test.go
@@ -0,0 +1,222 @@
+// Copyright 2024 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestGenerateChecksum(t *testing.T) {
+ tmpDir := t.TempDir()
+
+ // Define paths for the temporary files.
+ yamlFilePath := filepath.Join(tmpDir, "test.yml")
+ ruleFilePath := filepath.Join(tmpDir, "rule_file.yml")
+ scrapeConfigFilePath := filepath.Join(tmpDir, "scrape_config.yml")
+
+ // Define initial and modified content for the files.
+ originalRuleContent := "groups:\n- name: example\n rules:\n - alert: ExampleAlert"
+ modifiedRuleContent := "groups:\n- name: example\n rules:\n - alert: ModifiedAlert"
+
+ originalScrapeConfigContent := "scrape_configs:\n- job_name: example"
+ modifiedScrapeConfigContent := "scrape_configs:\n- job_name: modified_example"
+
+ // Define YAML content referencing the rule and scrape config files.
+ yamlContent := `
+rule_files:
+ - rule_file.yml
+scrape_config_files:
+ - scrape_config.yml
+`
+
+ // Write initial content to files.
+ require.NoError(t, os.WriteFile(ruleFilePath, []byte(originalRuleContent), 0o644))
+ require.NoError(t, os.WriteFile(scrapeConfigFilePath, []byte(originalScrapeConfigContent), 0o644))
+ require.NoError(t, os.WriteFile(yamlFilePath, []byte(yamlContent), 0o644))
+
+ // Generate the original checksum.
+ originalChecksum := calculateChecksum(t, yamlFilePath)
+
+ t.Run("Rule File Change", func(t *testing.T) {
+ // Modify the rule file.
+ require.NoError(t, os.WriteFile(ruleFilePath, []byte(modifiedRuleContent), 0o644))
+
+ // Checksum should change.
+ modifiedChecksum := calculateChecksum(t, yamlFilePath)
+ require.NotEqual(t, originalChecksum, modifiedChecksum)
+
+ // Revert the rule file.
+ require.NoError(t, os.WriteFile(ruleFilePath, []byte(originalRuleContent), 0o644))
+
+ // Checksum should return to the original.
+ revertedChecksum := calculateChecksum(t, yamlFilePath)
+ require.Equal(t, originalChecksum, revertedChecksum)
+ })
+
+ t.Run("Scrape Config Change", func(t *testing.T) {
+ // Modify the scrape config file.
+ require.NoError(t, os.WriteFile(scrapeConfigFilePath, []byte(modifiedScrapeConfigContent), 0o644))
+
+ // Checksum should change.
+ modifiedChecksum := calculateChecksum(t, yamlFilePath)
+ require.NotEqual(t, originalChecksum, modifiedChecksum)
+
+ // Revert the scrape config file.
+ require.NoError(t, os.WriteFile(scrapeConfigFilePath, []byte(originalScrapeConfigContent), 0o644))
+
+ // Checksum should return to the original.
+ revertedChecksum := calculateChecksum(t, yamlFilePath)
+ require.Equal(t, originalChecksum, revertedChecksum)
+ })
+
+ t.Run("Rule File Deletion", func(t *testing.T) {
+ // Delete the rule file.
+ require.NoError(t, os.Remove(ruleFilePath))
+
+ // Checksum should change.
+ deletedChecksum := calculateChecksum(t, yamlFilePath)
+ require.NotEqual(t, originalChecksum, deletedChecksum)
+
+ // Restore the rule file.
+ require.NoError(t, os.WriteFile(ruleFilePath, []byte(originalRuleContent), 0o644))
+
+ // Checksum should return to the original.
+ revertedChecksum := calculateChecksum(t, yamlFilePath)
+ require.Equal(t, originalChecksum, revertedChecksum)
+ })
+
+ t.Run("Scrape Config Deletion", func(t *testing.T) {
+ // Delete the scrape config file.
+ require.NoError(t, os.Remove(scrapeConfigFilePath))
+
+ // Checksum should change.
+ deletedChecksum := calculateChecksum(t, yamlFilePath)
+ require.NotEqual(t, originalChecksum, deletedChecksum)
+
+ // Restore the scrape config file.
+ require.NoError(t, os.WriteFile(scrapeConfigFilePath, []byte(originalScrapeConfigContent), 0o644))
+
+ // Checksum should return to the original.
+ revertedChecksum := calculateChecksum(t, yamlFilePath)
+ require.Equal(t, originalChecksum, revertedChecksum)
+ })
+
+ t.Run("Main File Change", func(t *testing.T) {
+ // Modify the main YAML file.
+ modifiedYamlContent := `
+global:
+ scrape_interval: 3s
+rule_files:
+ - rule_file.yml
+scrape_config_files:
+ - scrape_config.yml
+`
+ require.NoError(t, os.WriteFile(yamlFilePath, []byte(modifiedYamlContent), 0o644))
+
+ // Checksum should change.
+ modifiedChecksum := calculateChecksum(t, yamlFilePath)
+ require.NotEqual(t, originalChecksum, modifiedChecksum)
+
+ // Revert the main YAML file.
+ require.NoError(t, os.WriteFile(yamlFilePath, []byte(yamlContent), 0o644))
+
+ // Checksum should return to the original.
+ revertedChecksum := calculateChecksum(t, yamlFilePath)
+ require.Equal(t, originalChecksum, revertedChecksum)
+ })
+
+ t.Run("Rule File Removed from YAML Config", func(t *testing.T) {
+ // Modify the YAML content to remove the rule file.
+ modifiedYamlContent := `
+scrape_config_files:
+ - scrape_config.yml
+`
+ require.NoError(t, os.WriteFile(yamlFilePath, []byte(modifiedYamlContent), 0o644))
+
+ // Checksum should change.
+ modifiedChecksum := calculateChecksum(t, yamlFilePath)
+ require.NotEqual(t, originalChecksum, modifiedChecksum)
+
+ // Revert the YAML content.
+ require.NoError(t, os.WriteFile(yamlFilePath, []byte(yamlContent), 0o644))
+
+ // Checksum should return to the original.
+ revertedChecksum := calculateChecksum(t, yamlFilePath)
+ require.Equal(t, originalChecksum, revertedChecksum)
+ })
+
+ t.Run("Scrape Config Removed from YAML Config", func(t *testing.T) {
+ // Modify the YAML content to remove the scrape config file.
+ modifiedYamlContent := `
+rule_files:
+ - rule_file.yml
+`
+ require.NoError(t, os.WriteFile(yamlFilePath, []byte(modifiedYamlContent), 0o644))
+
+ // Checksum should change.
+ modifiedChecksum := calculateChecksum(t, yamlFilePath)
+ require.NotEqual(t, originalChecksum, modifiedChecksum)
+
+ // Revert the YAML content.
+ require.NoError(t, os.WriteFile(yamlFilePath, []byte(yamlContent), 0o644))
+
+ // Checksum should return to the original.
+ revertedChecksum := calculateChecksum(t, yamlFilePath)
+ require.Equal(t, originalChecksum, revertedChecksum)
+ })
+
+ t.Run("Empty Rule File", func(t *testing.T) {
+ // Write an empty rule file.
+ require.NoError(t, os.WriteFile(ruleFilePath, []byte(""), 0o644))
+
+ // Checksum should change.
+ emptyChecksum := calculateChecksum(t, yamlFilePath)
+ require.NotEqual(t, originalChecksum, emptyChecksum)
+
+ // Restore the rule file.
+ require.NoError(t, os.WriteFile(ruleFilePath, []byte(originalRuleContent), 0o644))
+
+ // Checksum should return to the original.
+ revertedChecksum := calculateChecksum(t, yamlFilePath)
+ require.Equal(t, originalChecksum, revertedChecksum)
+ })
+
+ t.Run("Empty Scrape Config File", func(t *testing.T) {
+ // Write an empty scrape config file.
+ require.NoError(t, os.WriteFile(scrapeConfigFilePath, []byte(""), 0o644))
+
+ // Checksum should change.
+ emptyChecksum := calculateChecksum(t, yamlFilePath)
+ require.NotEqual(t, originalChecksum, emptyChecksum)
+
+ // Restore the scrape config file.
+ require.NoError(t, os.WriteFile(scrapeConfigFilePath, []byte(originalScrapeConfigContent), 0o644))
+
+ // Checksum should return to the original.
+ revertedChecksum := calculateChecksum(t, yamlFilePath)
+ require.Equal(t, originalChecksum, revertedChecksum)
+ })
+}
+
+// calculateChecksum generates a checksum for the given YAML file path.
+func calculateChecksum(t *testing.T, yamlFilePath string) string {
+ checksum, err := GenerateChecksum(yamlFilePath)
+ require.NoError(t, err)
+ require.NotEmpty(t, checksum)
+ return checksum
+}
diff --git a/config/testdata/conf.good.yml b/config/testdata/conf.good.yml
index 184e6363ce..9eb7995432 100644
--- a/config/testdata/conf.good.yml
+++ b/config/testdata/conf.good.yml
@@ -8,6 +8,8 @@ global:
label_limit: 30
label_name_length_limit: 200
label_value_length_limit: 200
+ query_log_file: query.log
+ scrape_failure_log_file: fail.log
# scrape_timeout is set to the global default (10s).
external_labels:
@@ -37,6 +39,7 @@ remote_write:
key_file: valid_key_file
- url: http://remote2/push
+ protobuf_message: io.prometheus.write.v2.Request
name: rw_tls
tls_config:
cert_file: valid_cert_file
@@ -44,6 +47,9 @@ remote_write:
headers:
name: value
+otlp:
+ promote_resource_attributes: ["k8s.cluster.name", "k8s.job.name", "k8s.namespace.name"]
+
remote_read:
- url: http://remote1/read
read_recent: true
@@ -68,6 +74,7 @@ scrape_configs:
# metrics_path defaults to '/metrics'
# scheme defaults to 'http'.
+ scrape_failure_log_file: fail_prom.log
file_sd_configs:
- files:
- foo/*.slow.json
@@ -83,6 +90,12 @@ scrape_configs:
my: label
your: label
+ http_headers:
+ foo:
+ values: ["foobar"]
+ secrets: ["bar", "foo"]
+ files: ["valid_password_file"]
+
relabel_configs:
- source_labels: [job, __meta_dns_name]
regex: (.*)some-[regex]
diff --git a/config/testdata/jobname_dup.bad.yml b/config/testdata/jobname_dup.bad.yml
index 0265493c30..d03cb0cf97 100644
--- a/config/testdata/jobname_dup.bad.yml
+++ b/config/testdata/jobname_dup.bad.yml
@@ -1,4 +1,6 @@
# Two scrape configs with the same job names are not allowed.
+global:
+ metric_name_validation_scheme: legacy
scrape_configs:
- job_name: prometheus
- job_name: service-x
diff --git a/config/testdata/lowercase.bad.yml b/config/testdata/lowercase.bad.yml
index 9bc9583341..6dd72e6476 100644
--- a/config/testdata/lowercase.bad.yml
+++ b/config/testdata/lowercase.bad.yml
@@ -1,3 +1,5 @@
+global:
+ metric_name_validation_scheme: legacy
scrape_configs:
- job_name: prometheus
relabel_configs:
diff --git a/config/testdata/otlp_sanitize_resource_attributes.bad.yml b/config/testdata/otlp_sanitize_resource_attributes.bad.yml
new file mode 100644
index 0000000000..37ec5d1209
--- /dev/null
+++ b/config/testdata/otlp_sanitize_resource_attributes.bad.yml
@@ -0,0 +1,2 @@
+otlp:
+ promote_resource_attributes: ["k8s.cluster.name", " k8s.job.name ", "k8s.namespace.name", "k8s.job.name", ""]
diff --git a/config/testdata/otlp_sanitize_resource_attributes.good.yml b/config/testdata/otlp_sanitize_resource_attributes.good.yml
new file mode 100644
index 0000000000..67247e7743
--- /dev/null
+++ b/config/testdata/otlp_sanitize_resource_attributes.good.yml
@@ -0,0 +1,2 @@
+otlp:
+ promote_resource_attributes: ["k8s.cluster.name", " k8s.job.name ", "k8s.namespace.name"]
diff --git a/config/testdata/remote_write_wrong_msg.bad.yml b/config/testdata/remote_write_wrong_msg.bad.yml
new file mode 100644
index 0000000000..0918309540
--- /dev/null
+++ b/config/testdata/remote_write_wrong_msg.bad.yml
@@ -0,0 +1,3 @@
+remote_write:
+ - url: localhost:9090
+ protobuf_message: io.prometheus.writet.v2.Request # typo in 'write"
diff --git a/config/testdata/scrape_config_default_validation_mode.yml b/config/testdata/scrape_config_default_validation_mode.yml
new file mode 100644
index 0000000000..96680d6438
--- /dev/null
+++ b/config/testdata/scrape_config_default_validation_mode.yml
@@ -0,0 +1,2 @@
+scrape_configs:
+ - job_name: prometheus
diff --git a/config/testdata/scrape_config_global_validation_mode.yml b/config/testdata/scrape_config_global_validation_mode.yml
new file mode 100644
index 0000000000..e9b0618c70
--- /dev/null
+++ b/config/testdata/scrape_config_global_validation_mode.yml
@@ -0,0 +1,4 @@
+global:
+ metric_name_validation_scheme: legacy
+scrape_configs:
+ - job_name: prometheus
diff --git a/config/testdata/scrape_config_local_global_validation_mode.yml b/config/testdata/scrape_config_local_global_validation_mode.yml
new file mode 100644
index 0000000000..30b54834a5
--- /dev/null
+++ b/config/testdata/scrape_config_local_global_validation_mode.yml
@@ -0,0 +1,5 @@
+global:
+ metric_name_validation_scheme: legacy
+scrape_configs:
+ - job_name: prometheus
+ metric_name_validation_scheme: utf8
diff --git a/config/testdata/scrape_config_local_validation_mode.yml b/config/testdata/scrape_config_local_validation_mode.yml
new file mode 100644
index 0000000000..90279ff081
--- /dev/null
+++ b/config/testdata/scrape_config_local_validation_mode.yml
@@ -0,0 +1,3 @@
+scrape_configs:
+ - job_name: prometheus
+ metric_name_validation_scheme: legacy
diff --git a/console_libraries/menu.lib b/console_libraries/menu.lib
deleted file mode 100644
index 199ebf9f48..0000000000
--- a/console_libraries/menu.lib
+++ /dev/null
@@ -1,82 +0,0 @@
-{{/* vim: set ft=html: */}}
-
-{{/* Navbar, should be passed . */}}
-{{ define "navbar" }}
-
-{{ end }}
-
-{{/* LHS menu, should be passed . */}}
-{{ define "menu" }}
-