diff --git a/builtin/credential/aws/path_login.go b/builtin/credential/aws/path_login.go index fdb37da72b..46b575a10d 100644 --- a/builtin/credential/aws/path_login.go +++ b/builtin/credential/aws/path_login.go @@ -15,7 +15,6 @@ import ( "io" "net/http" "net/url" - "regexp" "strings" "time" @@ -30,6 +29,7 @@ import ( "github.com/hashicorp/go-retryablehttp" "github.com/hashicorp/go-secure-stdlib/awsutil" "github.com/hashicorp/go-secure-stdlib/parseutil" + iRegexp "github.com/hashicorp/go-secure-stdlib/regexp" "github.com/hashicorp/go-secure-stdlib/strutil" "github.com/hashicorp/go-uuid" "github.com/hashicorp/vault/helper/pkcs7" @@ -1678,8 +1678,10 @@ func validateVaultHeaderValue(method string, headers http.Header, parsedUrl *url case http.MethodPost: if authzHeaders, ok := headers["Authorization"]; ok { // authzHeader looks like AWS4-HMAC-SHA256 Credential=AKI..., SignedHeaders=host;x-amz-date;x-vault-awsiam-id, Signature=... - // We need to extract out the SignedHeaders - re := regexp.MustCompile(".*SignedHeaders=([^,]+)") + // We need to extract out the SignedHeaders. + // We are using an interned regexp library here to avoid redundant objects and save memory. + re := iRegexp.MustCompileInterned(".*SignedHeaders=([^,]+)") + authzHeader := strings.Join(authzHeaders, ",") matches := re.FindSubmatch([]byte(authzHeader)) if len(matches) < 1 { diff --git a/changelog/31022.txt b/changelog/31022.txt new file mode 100644 index 0000000000..737ce1a011 --- /dev/null +++ b/changelog/31022.txt @@ -0,0 +1,3 @@ +```release-note:improvement +core: Improve memory use of path management for namespaces, auth methods, and secrets engines. Now Vault should handle larger numbers of namespaces and multiple instances of the same secrets engine or auth method more efficiently. +``` diff --git a/go.mod b/go.mod index 214568af8f..71cbe02e0a 100644 --- a/go.mod +++ b/go.mod @@ -116,6 +116,7 @@ require ( github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 github.com/hashicorp/go-secure-stdlib/password v0.1.1 github.com/hashicorp/go-secure-stdlib/permitpool v1.0.0 + github.com/hashicorp/go-secure-stdlib/regexp v1.0.0 github.com/hashicorp/go-secure-stdlib/reloadutil v0.1.1 github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 github.com/hashicorp/go-secure-stdlib/tlsutil v0.1.3 diff --git a/go.sum b/go.sum index 18e3749360..caa8469216 100644 --- a/go.sum +++ b/go.sum @@ -1479,6 +1479,8 @@ github.com/hashicorp/go-secure-stdlib/permitpool v1.0.0 h1:U6y5MXGiDVOOtkWJ6o/tu github.com/hashicorp/go-secure-stdlib/permitpool v1.0.0/go.mod h1:ecDb3o+8D4xtP0nTCufJaAVawHavy5M2eZ64Nq/8/LM= github.com/hashicorp/go-secure-stdlib/plugincontainer v0.4.1 h1:JY+zGg8gOmslwif1fiCqT5Hu1SikLZQcHkmQhCoA9gY= github.com/hashicorp/go-secure-stdlib/plugincontainer v0.4.1/go.mod h1:jW3KCTvdPyAdVecOUwiiO2XaYgUJ/isigt++ISkszkY= +github.com/hashicorp/go-secure-stdlib/regexp v1.0.0 h1:08mz6j5MsCG9sf8tvC8Lhboe/ZMiNg41IPSh6unK5T4= +github.com/hashicorp/go-secure-stdlib/regexp v1.0.0/go.mod h1:n/Gj3sYIEEOYds8uKS55bFf7XiYvWN4e+d+UOA7r/YU= github.com/hashicorp/go-secure-stdlib/reloadutil v0.1.1 h1:SMGUnbpAcat8rIKHkBPjfv81yC46a8eCNZ2hsR2l1EI= github.com/hashicorp/go-secure-stdlib/reloadutil v0.1.1/go.mod h1:Ch/bf00Qnx77MZd49JRgHYqHQjtEmTgGU2faufpVZb0= github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts= diff --git a/sdk/framework/backend.go b/sdk/framework/backend.go index 54f426772d..e2e96c8719 100644 --- a/sdk/framework/backend.go +++ b/sdk/framework/backend.go @@ -24,6 +24,7 @@ import ( "github.com/hashicorp/go-kms-wrapping/entropy/v2" "github.com/hashicorp/go-multierror" "github.com/hashicorp/go-secure-stdlib/parseutil" + iRegexp "github.com/hashicorp/go-secure-stdlib/regexp" "github.com/hashicorp/vault/sdk/database/helper/connutil" "github.com/hashicorp/vault/sdk/helper/consts" "github.com/hashicorp/vault/sdk/helper/errutil" @@ -537,8 +538,10 @@ func (b *Backend) init() { p.Pattern = p.Pattern + "$" } - // Detect the coding error of an invalid Pattern - b.pathsRe[i] = regexp.MustCompile(p.Pattern) + // Detect the coding error of an invalid Pattern. We are using an interned + // regexps library here to save memory, since we can have many instances of the + // same backend. + b.pathsRe[i] = iRegexp.MustCompileInterned(p.Pattern) } } diff --git a/sdk/go.mod b/sdk/go.mod index 45cb8f749b..77763d5913 100644 --- a/sdk/go.mod +++ b/sdk/go.mod @@ -35,6 +35,7 @@ require ( github.com/hashicorp/go-secure-stdlib/password v0.1.1 github.com/hashicorp/go-secure-stdlib/permitpool v1.0.0 github.com/hashicorp/go-secure-stdlib/plugincontainer v0.4.1 + github.com/hashicorp/go-secure-stdlib/regexp v1.0.0 github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 github.com/hashicorp/go-secure-stdlib/tlsutil v0.1.3 github.com/hashicorp/go-sockaddr v1.0.7 diff --git a/sdk/go.sum b/sdk/go.sum index 3ab8d68298..7017abdc87 100644 --- a/sdk/go.sum +++ b/sdk/go.sum @@ -215,6 +215,8 @@ github.com/hashicorp/go-secure-stdlib/permitpool v1.0.0 h1:U6y5MXGiDVOOtkWJ6o/tu github.com/hashicorp/go-secure-stdlib/permitpool v1.0.0/go.mod h1:ecDb3o+8D4xtP0nTCufJaAVawHavy5M2eZ64Nq/8/LM= github.com/hashicorp/go-secure-stdlib/plugincontainer v0.4.1 h1:JY+zGg8gOmslwif1fiCqT5Hu1SikLZQcHkmQhCoA9gY= github.com/hashicorp/go-secure-stdlib/plugincontainer v0.4.1/go.mod h1:jW3KCTvdPyAdVecOUwiiO2XaYgUJ/isigt++ISkszkY= +github.com/hashicorp/go-secure-stdlib/regexp v1.0.0 h1:08mz6j5MsCG9sf8tvC8Lhboe/ZMiNg41IPSh6unK5T4= +github.com/hashicorp/go-secure-stdlib/regexp v1.0.0/go.mod h1:n/Gj3sYIEEOYds8uKS55bFf7XiYvWN4e+d+UOA7r/YU= github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts= github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4= github.com/hashicorp/go-secure-stdlib/tlsutil v0.1.3 h1:xbrxd0U9XQW8qL1BAz2XrAjAF/P2vcqUTAues9c24B8= diff --git a/vault/external_tests/interned_regexp/interned_regexp_test.go b/vault/external_tests/interned_regexp/interned_regexp_test.go new file mode 100644 index 0000000000..a86be5449d --- /dev/null +++ b/vault/external_tests/interned_regexp/interned_regexp_test.go @@ -0,0 +1,50 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package loadedsnapshots + +import ( + "fmt" + "strconv" + "sync" + "testing" + + "github.com/hashicorp/vault/api" + "github.com/hashicorp/vault/helper/testhelpers/minimal" + "github.com/stretchr/testify/require" +) + +// TestInternedRegexpConcurrentAccess tests that multiple goroutines can +// concurrently access and create multiple instances of the same type of mount +// without causing any issues. This is important to ensure that the interned +// regular expressions used in the mount paths do not cause have concurrency +// faults. +func TestInternedRegexpConcurrentAccess(t *testing.T) { + cluster := minimal.NewTestSoloCluster(t, nil) + client := cluster.Cores[0].Client + + // Build mount input + mountInput := &api.MountInput{ + Type: "pki", + } + + // Mount 10 PKI secrets engine mounts. + wg := sync.WaitGroup{} + for i := 0; i < 10; i++ { + mountPath := "pki" + strconv.Itoa(i) + wg.Add(1) + go func() { + err := client.Sys().Mount(mountPath, mountInput) + require.NoError(t, err) + + // Verify the mount was created + _, err = client.Sys().GetMount(mountPath) + require.NoError(t, err) + + _, err = client.Logical().List(fmt.Sprintf("/%s/roles", mountPath)) + require.NoError(t, err) + wg.Done() + }() + } + wg.Wait() +}