vault/command/agentproxyshared/cache/static_secret_cache_updater_test.go

// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: BUSL-1.1

package cache

import (
	"context"
	"fmt"
	"sync"
	"testing"
	"time"

	"github.com/hashicorp/go-hclog"
	kv "github.com/hashicorp/vault-plugin-secrets-kv"
	"github.com/hashicorp/vault/api"
	"github.com/hashicorp/vault/command/agentproxyshared/cache/cacheboltdb"
	"github.com/hashicorp/vault/command/agentproxyshared/cache/cachememdb"
	"github.com/hashicorp/vault/command/agentproxyshared/sink"
	"github.com/hashicorp/vault/helper/testhelpers/minimal"
	vaulthttp "github.com/hashicorp/vault/http"
	"github.com/hashicorp/vault/sdk/helper/logging"
	"github.com/hashicorp/vault/sdk/logical"
	"github.com/hashicorp/vault/vault"
	"github.com/stretchr/testify/require"
	"go.uber.org/atomic"
	"nhooyr.io/websocket"
)

// Avoiding a circular dependency in the test.
type mockSink struct {
	token *atomic.String
}

func (m *mockSink) Token() string {
	return m.token.Load()
}

func (m *mockSink) WriteToken(token string) error {
	m.token.Store(token)
	return nil
}

func newMockSink(t *testing.T) sink.Sink {
	t.Helper()

	return &mockSink{
		token: atomic.NewString(""),
	}
}

// testNewStaticSecretCacheUpdater returns a new StaticSecretCacheUpdater
// for use in tests.
func testNewStaticSecretCacheUpdater(t *testing.T, client *api.Client) *StaticSecretCacheUpdater {
	t.Helper()

	lc := testNewLeaseCache(t, []*SendResponse{})
	tokenSink := newMockSink(t)
	tokenSink.WriteToken(client.Token())

	updater, err := NewStaticSecretCacheUpdater(&StaticSecretCacheUpdaterConfig{
		Client:     client,
		LeaseCache: lc,
		Logger:     logging.NewVaultLogger(hclog.Trace).Named("cache.updater"),
		TokenSink:  tokenSink,
	})
	if err != nil {
		t.Fatal(err)
	}
	return updater
}

// TestNewStaticSecretCacheUpdater tests the NewStaticSecretCacheUpdater method,
// to ensure it errors out when appropriate.
func TestNewStaticSecretCacheUpdater(t *testing.T) {
	t.Parallel()

	lc := testNewLeaseCache(t, []*SendResponse{})
	config := api.DefaultConfig()
	logger := logging.NewVaultLogger(hclog.Trace).Named("cache.updater")
	client, err := api.NewClient(config)
	if err != nil {
		t.Fatal(err)
	}
	tokenSink := newMockSink(t)

	// Expect an error if any of the arguments are nil:
	updater, err := NewStaticSecretCacheUpdater(&StaticSecretCacheUpdaterConfig{
		Client:     nil,
		LeaseCache: lc,
		Logger:     logger,
		TokenSink:  tokenSink,
	})
	require.Error(t, err)
	require.Nil(t, updater)

	updater, err = NewStaticSecretCacheUpdater(&StaticSecretCacheUpdaterConfig{
		Client:     client,
		LeaseCache: nil,
		Logger:     logger,
		TokenSink:  tokenSink,
	})
	require.Error(t, err)
	require.Nil(t, updater)

	updater, err = NewStaticSecretCacheUpdater(&StaticSecretCacheUpdaterConfig{
		Client:     client,
		LeaseCache: lc,
		Logger:     nil,
		TokenSink:  tokenSink,
	})
	require.Error(t, err)
	require.Nil(t, updater)

	updater, err = NewStaticSecretCacheUpdater(&StaticSecretCacheUpdaterConfig{
		Client:     client,
		LeaseCache: lc,
		Logger:     logging.NewVaultLogger(hclog.Trace).Named("cache.updater"),
		TokenSink:  nil,
	})
	require.Error(t, err)
	require.Nil(t, updater)

	// Don't expect an error if the arguments are as expected
	updater, err = NewStaticSecretCacheUpdater(&StaticSecretCacheUpdaterConfig{
		Client:     client,
		LeaseCache: lc,
		Logger:     logging.NewVaultLogger(hclog.Trace).Named("cache.updater"),
		TokenSink:  tokenSink,
	})
	if err != nil {
		t.Fatal(err)
	}
	require.NotNil(t, updater)
}

// TestOpenWebSocketConnection tests that the openWebSocketConnection function
// works as expected. This uses a TLS enabled (wss) WebSocket connection.
func TestOpenWebSocketConnection(t *testing.T) {
	t.Parallel()
	// We need a valid cluster for the connection to succeed.
	cluster := minimal.NewTestSoloCluster(t, nil)
	client := cluster.Cores[0].Client

	updater := testNewStaticSecretCacheUpdater(t, client)
	updater.tokenSink.WriteToken(client.Token())

	conn, err := updater.openWebSocketConnection(context.Background())
	if err != nil {
		t.Fatal(err)
	}
	require.NotNil(t, conn)
}

// TestOpenWebSocketConnectionReceivesEventsDefaultMount tests that the openWebSocketConnection function
// works as expected with the default KVV1 mount, and then the connection can be used to receive an event.
// This acts as more of an event system sanity check than a test of the updater
// logic. It's still important coverage, though.
// As of right now, it does not pass since the default kv mount is LeasedPassthroughBackend.
// If that is changed, this test will be unskipped.
func TestOpenWebSocketConnectionReceivesEventsDefaultMount(t *testing.T) {
	t.Parallel()
	t.Skip("This test won't finish, as the default KV mount is LeasedPassthroughBackend in tests, and therefore does not send events")
	// We need a valid cluster for the connection to succeed.
	cluster := vault.NewTestCluster(t, nil, &vault.TestClusterOptions{
		HandlerFunc: vaulthttp.Handler,
	})
	client := cluster.Cores[0].Client

	updater := testNewStaticSecretCacheUpdater(t, client)

	conn, err := updater.openWebSocketConnection(context.Background())
	if err != nil {
		t.Fatal(err)
	}
	require.NotNil(t, conn)

	t.Cleanup(func() {
		conn.Close(websocket.StatusNormalClosure, "")
	})

	makeData := func(i int) map[string]interface{} {
		return map[string]interface{}{
			"foo": fmt.Sprintf("bar%d", i),
		}
	}
	// Put a secret, which should trigger an event
	err = client.KVv1("secret").Put(context.Background(), "foo", makeData(100))
	if err != nil {
		t.Fatal(err)
	}

	for i := 0; i < 5; i++ {
		// Do a fresh PUT just to refresh the secret and send a new message
		err = client.KVv1("secret").Put(context.Background(), "foo", makeData(i))
		if err != nil {
			t.Fatal(err)
		}

		// This method blocks until it gets a secret, so this test
		// will only pass if we're receiving events correctly.
		_, message, err := conn.Read(context.Background())
		if err != nil {
			t.Fatal(err)
		}
		t.Log(string(message))
	}
}

// TestOpenWebSocketConnectionReceivesEventsKVV1 tests that the openWebSocketConnection function
// works as expected with KVV1, and then the connection can be used to receive an event.
// This acts as more of an event system sanity check than a test of the updater
// logic. It's still important coverage, though.
func TestOpenWebSocketConnectionReceivesEventsKVV1(t *testing.T) {
	t.Parallel()
	// We need a valid cluster for the connection to succeed.
	cluster := vault.NewTestCluster(t, &vault.CoreConfig{
		LogicalBackends: map[string]logical.Factory{
			"kv": kv.Factory,
		},
	}, &vault.TestClusterOptions{
		HandlerFunc: vaulthttp.Handler,
	})
	client := cluster.Cores[0].Client

	updater := testNewStaticSecretCacheUpdater(t, client)

	conn, err := updater.openWebSocketConnection(context.Background())
	if err != nil {
		t.Fatal(err)
	}
	require.NotNil(t, conn)

	t.Cleanup(func() {
		conn.Close(websocket.StatusNormalClosure, "")
	})

	err = client.Sys().Mount("secret-v1", &api.MountInput{
		Type: "kv",
	})
	if err != nil {
		t.Fatal(err)
	}

	makeData := func(i int) map[string]interface{} {
		return map[string]interface{}{
			"foo": fmt.Sprintf("bar%d", i),
		}
	}
	// Put a secret, which should trigger an event
	err = client.KVv1("secret-v1").Put(context.Background(), "foo", makeData(100))
	if err != nil {
		t.Fatal(err)
	}

	for i := 0; i < 5; i++ {
		// Do a fresh PUT just to refresh the secret and send a new message
		err = client.KVv1("secret-v1").Put(context.Background(), "foo", makeData(i))
		if err != nil {
			t.Fatal(err)
		}

		// This method blocks until it gets a secret, so this test
		// will only pass if we're receiving events correctly.
		_, _, err := conn.Read(context.Background())
		if err != nil {
			t.Fatal(err)
		}
	}
}

// TestOpenWebSocketConnectionReceivesEvents tests that the openWebSocketConnection function
// works as expected with KVV2, and then the connection can be used to receive an event.
// This acts as more of an event system sanity check than a test of the updater
// logic. It's still important coverage, though.
func TestOpenWebSocketConnectionReceivesEventsKVV2(t *testing.T) {
	t.Parallel()
	// We need a valid cluster for the connection to succeed.
	cluster := vault.NewTestCluster(t, &vault.CoreConfig{
		LogicalBackends: map[string]logical.Factory{
			"kv": kv.VersionedKVFactory,
		},
	}, &vault.TestClusterOptions{
		HandlerFunc: vaulthttp.Handler,
	})
	client := cluster.Cores[0].Client

	updater := testNewStaticSecretCacheUpdater(t, client)

	conn, err := updater.openWebSocketConnection(context.Background())
	if err != nil {
		t.Fatal(err)
	}
	require.NotNil(t, conn)

	t.Cleanup(func() {
		conn.Close(websocket.StatusNormalClosure, "")
	})

	makeData := func(i int) map[string]interface{} {
		return map[string]interface{}{
			"foo": fmt.Sprintf("bar%d", i),
		}
	}

	err = client.Sys().Mount("secret-v2", &api.MountInput{
		Type: "kv-v2",
	})
	if err != nil {
		t.Fatal(err)
	}

	// Put a secret, which should trigger an event
	_, err = client.KVv2("secret-v2").Put(context.Background(), "foo", makeData(100))
	if err != nil {
		t.Fatal(err)
	}

	for i := 0; i < 5; i++ {
		// Do a fresh PUT just to refresh the secret and send a new message
		_, err = client.KVv2("secret-v2").Put(context.Background(), "foo", makeData(i))
		if err != nil {
			t.Fatal(err)
		}

		// This method blocks until it gets a secret, so this test
		// will only pass if we're receiving events correctly.
		_, _, err := conn.Read(context.Background())
		if err != nil {
			t.Fatal(err)
		}
	}
}

// TestOpenWebSocketConnectionTestServer tests that the openWebSocketConnection function
// works as expected using vaulthttp.TestServer. This server isn't TLS enabled, so tests
// the ws path (as opposed to the wss) path.
func TestOpenWebSocketConnectionTestServer(t *testing.T) {
	t.Parallel()
	// We need a valid cluster for the connection to succeed.
	core := vault.TestCoreWithConfig(t, &vault.CoreConfig{})
	ln, addr := vaulthttp.TestServer(t, core)
	defer ln.Close()

	keys, rootToken := vault.TestCoreInit(t, core)
	for _, key := range keys {
		_, err := core.Unseal(key)
		if err != nil {
			t.Fatal(err)
		}
	}

	config := api.DefaultConfig()
	config.Address = addr
	client, err := api.NewClient(config)
	if err != nil {
		t.Fatal(err)
	}
	client.SetToken(rootToken)
	updater := testNewStaticSecretCacheUpdater(t, client)

	conn, err := updater.openWebSocketConnection(context.Background())
	if err != nil {
		t.Fatal(err)
	}
	require.NotNil(t, conn)
}

// Test_StreamStaticSecretEvents_UpdatesCacheWithNewSecrets tests that an event will
// properly update the corresponding secret in Proxy's cache. This is a little more end-to-end-y
// than TestUpdateStaticSecret, and essentially is testing a similar thing, though is
// ensuring that updateStaticSecret gets called by the event arriving
// (as part of streamStaticSecretEvents) instead of testing calling it explicitly.
func Test_StreamStaticSecretEvents_UpdatesCacheWithNewSecrets(t *testing.T) {
	t.Parallel()
	cluster := vault.NewTestCluster(t, &vault.CoreConfig{
		LogicalBackends: map[string]logical.Factory{
			"kv": kv.VersionedKVFactory,
		},
	}, &vault.TestClusterOptions{
		HandlerFunc: vaulthttp.Handler,
	})
	client := cluster.Cores[0].Client

	updater := testNewStaticSecretCacheUpdater(t, client)
	leaseCache := updater.leaseCache

	wg := &sync.WaitGroup{}
	runStreamStaticSecretEvents := func() {
		wg.Add(1)
		err := updater.streamStaticSecretEvents(context.Background())
		if err != nil {
			t.Fatal(err)
		}
	}
	go runStreamStaticSecretEvents()

	// First, create the secret in the cache that we expect to be updated:
	path := "secret-v2/data/foo"
	indexId := hashStaticSecretIndex(path)
	initialTime := time.Now().UTC()
	// pre-populate the leaseCache with a secret to update
	index := &cachememdb.Index{
		Namespace:   "root/",
		RequestPath: path,
		LastRenewed: initialTime,
		ID:          indexId,
		// Valid token provided, so update should work.
		Tokens:   map[string]struct{}{client.Token(): {}},
		Response: []byte{},
	}
	err := leaseCache.db.Set(index)
	if err != nil {
		t.Fatal(err)
	}

	secretData := map[string]interface{}{
		"foo": "bar",
	}

	err = client.Sys().Mount("secret-v2", &api.MountInput{
		Type: "kv-v2",
	})
	if err != nil {
		t.Fatal(err)
	}

	// Wait for the event stream to be fully up and running. Should be faster than this in reality, but
	// we make it five seconds to protect against CI flakiness.
	time.Sleep(5 * time.Second)

	// Put a secret, which should trigger an event
	_, err = client.KVv2("secret-v2").Put(context.Background(), "foo", secretData)
	if err != nil {
		t.Fatal(err)
	}

	// Wait for the event to arrive. Events are usually much, much faster
	// than this, but we make it five seconds to protect against CI flakiness.
	time.Sleep(5 * time.Second)

	// Then, do a GET to see if the index got updated by the event
	newIndex, err := leaseCache.db.Get(cachememdb.IndexNameID, indexId)
	if err != nil {
		t.Fatal(err)
	}
	require.NotNil(t, newIndex)
	require.NotEqual(t, []byte{}, newIndex.Response)
	require.Truef(t, initialTime.Before(newIndex.LastRenewed), "last updated time not updated on index")
	require.Equal(t, index.RequestPath, newIndex.RequestPath)
	require.Equal(t, index.Tokens, newIndex.Tokens)

	wg.Done()
}

// TestUpdateStaticSecret tests that updateStaticSecret works as expected, reaching out
// to Vault to get an updated secret when called.
func TestUpdateStaticSecret(t *testing.T) {
	t.Parallel()
	// We need a valid cluster for the connection to succeed.
	cluster := vault.NewTestCluster(t, nil, &vault.TestClusterOptions{
		HandlerFunc: vaulthttp.Handler,
	})
	client := cluster.Cores[0].Client

	updater := testNewStaticSecretCacheUpdater(t, client)
	leaseCache := updater.leaseCache

	path := "secret/foo"
	indexId := hashStaticSecretIndex(path)
	initialTime := time.Now().UTC()
	// pre-populate the leaseCache with a secret to update
	index := &cachememdb.Index{
		Namespace:   "root/",
		RequestPath: "secret/foo",
		LastRenewed: initialTime,
		ID:          indexId,
		// Valid token provided, so update should work.
		Tokens:   map[string]struct{}{client.Token(): {}},
		Response: []byte{},
	}
	err := leaseCache.db.Set(index)
	if err != nil {
		t.Fatal(err)
	}

	secretData := map[string]interface{}{
		"foo": "bar",
	}

	// create the secret in Vault. n.b. the test cluster has already mounted the KVv1 backend at "secret"
	err = client.KVv1("secret").Put(context.Background(), "foo", secretData)
	if err != nil {
		t.Fatal(err)
	}

	// attempt the update
	err = updater.updateStaticSecret(context.Background(), path)
	if err != nil {
		t.Fatal(err)
	}

	newIndex, err := leaseCache.db.Get(cachememdb.IndexNameID, indexId)
	if err != nil {
		t.Fatal(err)
	}
	require.NotNil(t, newIndex)
	require.Truef(t, initialTime.Before(newIndex.LastRenewed), "last updated time not updated on index")
	require.NotEqual(t, []byte{}, newIndex.Response)
	require.Equal(t, index.RequestPath, newIndex.RequestPath)
	require.Equal(t, index.Tokens, newIndex.Tokens)
}

// TestUpdateStaticSecret_EvictsIfInvalidTokens tests that updateStaticSecret will
// evict secrets from the cache if no valid tokens are left.
func TestUpdateStaticSecret_EvictsIfInvalidTokens(t *testing.T) {
	t.Parallel()
	// We need a valid cluster for the connection to succeed.
	cluster := vault.NewTestCluster(t, nil, &vault.TestClusterOptions{
		HandlerFunc: vaulthttp.Handler,
	})
	client := cluster.Cores[0].Client

	updater := testNewStaticSecretCacheUpdater(t, client)
	leaseCache := updater.leaseCache

	path := "secret/foo"
	indexId := hashStaticSecretIndex(path)
	renewTime := time.Now().UTC()

	// pre-populate the leaseCache with a secret to update
	index := &cachememdb.Index{
		Namespace:   "root/",
		RequestPath: "secret/foo",
		LastRenewed: renewTime,
		ID:          indexId,
		// Note: invalid Tokens value provided, so this secret cannot be updated, and must be evicted
		Tokens: map[string]struct{}{"invalid token": {}},
	}
	err := leaseCache.db.Set(index)
	if err != nil {
		t.Fatal(err)
	}

	secretData := map[string]interface{}{
		"foo": "bar",
	}

	// create the secret in Vault. n.b. the test cluster has already mounted the KVv1 backend at "secret"
	err = client.KVv1("secret").Put(context.Background(), "foo", secretData)
	if err != nil {
		t.Fatal(err)
	}

	// attempt the update
	err = updater.updateStaticSecret(context.Background(), path)
	if err != nil {
		t.Fatal(err)
	}

	newIndex, err := leaseCache.db.Get(cachememdb.IndexNameID, indexId)
	require.Equal(t, cachememdb.ErrCacheItemNotFound, err)
	require.Nil(t, newIndex)
}

// TestUpdateStaticSecret_HandlesNonCachedPaths tests that updateStaticSecret
// doesn't fail or error if we try and give it an update to a path that isn't cached.
func TestUpdateStaticSecret_HandlesNonCachedPaths(t *testing.T) {
	t.Parallel()
	// We need a valid cluster for the connection to succeed.
	cluster := vault.NewTestCluster(t, nil, &vault.TestClusterOptions{
		HandlerFunc: vaulthttp.Handler,
	})
	client := cluster.Cores[0].Client

	updater := testNewStaticSecretCacheUpdater(t, client)

	path := "secret/foo"

	// attempt the update
	err := updater.updateStaticSecret(context.Background(), path)
	if err != nil {
		t.Fatal(err)
	}
	require.Nil(t, err)
}

// TestPreEventStreamUpdate tests that preEventStreamUpdate correctly
// updates old static secrets in the cache.
func TestPreEventStreamUpdate(t *testing.T) {
	t.Parallel()
	cluster := vault.NewTestCluster(t, &vault.CoreConfig{
		LogicalBackends: map[string]logical.Factory{
			"kv": kv.VersionedKVFactory,
		},
	}, &vault.TestClusterOptions{
		HandlerFunc: vaulthttp.Handler,
	})
	client := cluster.Cores[0].Client

	updater := testNewStaticSecretCacheUpdater(t, client)
	leaseCache := updater.leaseCache

	// First, create the secret in the cache that we expect to be updated:
	path := "secret-v2/data/foo"
	indexId := hashStaticSecretIndex(path)
	initialTime := time.Now().UTC()
	// pre-populate the leaseCache with a secret to update
	index := &cachememdb.Index{
		Namespace:   "root/",
		RequestPath: path,
		LastRenewed: initialTime,
		ID:          indexId,
		// Valid token provided, so update should work.
		Tokens:   map[string]struct{}{client.Token(): {}},
		Response: []byte{},
		Type:     cacheboltdb.StaticSecretType,
	}
	err := leaseCache.db.Set(index)
	if err != nil {
		t.Fatal(err)
	}

	secretData := map[string]interface{}{
		"foo": "bar",
	}

	err = client.Sys().Mount("secret-v2", &api.MountInput{
		Type: "kv-v2",
	})
	if err != nil {
		t.Fatal(err)
	}

	// Put a secret (with different values to what's currently in the cache)
	_, err = client.KVv2("secret-v2").Put(context.Background(), "foo", secretData)
	if err != nil {
		t.Fatal(err)
	}

	// perform the pre-event stream update:
	err = updater.preEventStreamUpdate(context.Background())
	require.Nil(t, err)

	// Then, do a GET to see if the event got updated
	newIndex, err := leaseCache.db.Get(cachememdb.IndexNameID, indexId)
	require.Nil(t, err)
	require.NotNil(t, newIndex)
	require.NotEqual(t, []byte{}, newIndex.Response)
	require.Truef(t, initialTime.Before(newIndex.LastRenewed), "last updated time not updated on index")
	require.Equal(t, index.RequestPath, newIndex.RequestPath)
	require.Equal(t, index.Tokens, newIndex.Tokens)
}

// TestPreEventStreamUpdateErrorUpdating tests that preEventStreamUpdate correctly responds
// to errors on secret updates
func TestPreEventStreamUpdateErrorUpdating(t *testing.T) {
	t.Parallel()
	cluster := vault.NewTestCluster(t, &vault.CoreConfig{
		LogicalBackends: map[string]logical.Factory{
			"kv": kv.VersionedKVFactory,
		},
	}, &vault.TestClusterOptions{
		HandlerFunc: vaulthttp.Handler,
	})
	client := cluster.Cores[0].Client

	updater := testNewStaticSecretCacheUpdater(t, client)
	leaseCache := updater.leaseCache

	// First, create the secret in the cache that we expect to be updated:
	path := "secret-v2/data/foo"
	indexId := hashStaticSecretIndex(path)
	initialTime := time.Now().UTC()
	// pre-populate the leaseCache with a secret to update
	index := &cachememdb.Index{
		Namespace:   "root/",
		RequestPath: path,
		LastRenewed: initialTime,
		ID:          indexId,
		// Valid token provided, so update should work.
		Tokens:   map[string]struct{}{client.Token(): {}},
		Response: []byte{},
		Type:     cacheboltdb.StaticSecretType,
	}
	err := leaseCache.db.Set(index)
	if err != nil {
		t.Fatal(err)
	}

	secretData := map[string]interface{}{
		"foo": "bar",
	}

	err = client.Sys().Mount("secret-v2", &api.MountInput{
		Type: "kv-v2",
	})
	if err != nil {
		t.Fatal(err)
	}

	// Put a secret (with different values to what's currently in the cache)
	_, err = client.KVv2("secret-v2").Put(context.Background(), "foo", secretData)
	if err != nil {
		t.Fatal(err)
	}

	// Seal Vault, so that the update will fail
	cluster.EnsureCoresSealed(t)

	// perform the pre-event stream update:
	err = updater.preEventStreamUpdate(context.Background())
	require.Nil(t, err)

	// Then, we expect the index to be evicted since the token failed to update
	_, err = leaseCache.db.Get(cachememdb.IndexNameID, indexId)
	require.Equal(t, cachememdb.ErrCacheItemNotFound, err)
}