OSS side barrier encryption tracking and automatic rotation (#11007)

* Automatic barrier key rotation, OSS portion

* Fix build issues

* Vendored version

* Add missing encs field, not sure where this got lost.

api/sys_rotate.go

@@ -68,10 +68,24 @@ func (c *Sys) KeyStatus() (*KeyStatus, error) {
 	}
 	result.InstallTime = installTime
 
+	encryptionsRaw, ok := secret.Data["encryptions"]
+	if ok {
+		encryptions, ok := encryptionsRaw.(json.Number)
+		if !ok {
+			return nil, errors.New("could not convert encryptions to a number")
+		}
+		encryptions64, err := encryptions.Int64()
+		if err != nil {
+			return nil, err
+		}
+		result.Encryptions = int(encryptions64)
+	}
+
 	return &result, err
 }
 
 type KeyStatus struct {
 	Term        int       `json:"term"`
 	InstallTime time.Time `json:"install_time"`
+	Encryptions int       `json:"encryptions"`
 }

http/sys_rotate_test.go

@@ -36,15 +36,16 @@ func TestSysRotate(t *testing.T) {
 	testResponseStatus(t, resp, 200)
 	testResponseBody(t, resp, &actual)
 
-	actualInstallTime, ok := actual["data"].(map[string]interface{})["install_time"]
-	if !ok || actualInstallTime == "" {
-		t.Fatal("install_time missing in data")
+	for _, field := range []string{"install_time", "encryptions"} {
+		actualVal, ok := actual["data"].(map[string]interface{})[field]
+		if !ok || actualVal == "" {
+			t.Fatal(field, " missing in data")
+		}
+		expected["data"].(map[string]interface{})[field] = actualVal
+		expected[field] = actualVal
 	}
-	expected["data"].(map[string]interface{})["install_time"] = actualInstallTime
-	expected["install_time"] = actualInstallTime
 
 	expected["request_id"] = actual["request_id"]
-
 	if diff := deep.Equal(actual, expected); diff != nil {
 		t.Fatal(diff)
 	}

sdk/physical/file/file.go

@@ -225,9 +225,14 @@ func (b *FileBackend) PutInternal(ctx context.Context, entry *physical.Entry) er
 	if err := b.validatePath(entry.Key); err != nil {
 		return err
 	}
-
 	path, key := b.expandPath(entry.Key)
 
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	default:
+	}
+
 	// Make the parent tree
 	if err := os.MkdirAll(path, 0700); err != nil {
 		return err
@@ -249,12 +254,6 @@ func (b *FileBackend) PutInternal(ctx context.Context, entry *physical.Entry) er
 		return errors.New("could not successfully get a file handle")
 	}
 
-	select {
-	case <-ctx.Done():
-		return ctx.Err()
-	default:
-	}
-
 	enc := json.NewEncoder(f)
 	encErr := enc.Encode(&fileEntry{
 		Value: entry.Value,

vault/barrier.go

@@ -136,12 +136,28 @@ type SecurityBarrier interface {
 	// ActiveKeyInfo is used to inform details about the active key
 	ActiveKeyInfo() (*KeyInfo, error)
 
+	// RotationConfig returns the auto-rotation config for the barrier key
+	RotationConfig() (KeyRotationConfig, error)
+
+	// SetRotationConfig updates the auto-rotation config for the barrier key
+	SetRotationConfig(ctx context.Context, config KeyRotationConfig) error
+
 	// Rekey is used to change the master key used to protect the keyring
 	Rekey(context.Context, []byte) error
 
 	// For replication we must send over the keyring, so this must be available
 	Keyring() (*Keyring, error)
 
+	// For encryption count shipping, a function which handles updating local encryption counts if the consumer succeeds.
+	// This isolates the barrier code from the replication system
+	ConsumeEncryptionCount(consumer func(int64) error) error
+
+	// Add encryption counts from a remote source (downstream cluster node)
+	AddRemoteEncryptions(encryptions int64)
+
+	// Check whether an automatic rotation is due
+	CheckBarrierAutoRotate(ctx context.Context) (string, error)
+
 	// SecurityBarrier must provide the storage APIs
 	logical.Storage
 
@@ -177,4 +193,5 @@ type BarrierEncryptor interface {
 type KeyInfo struct {
 	Term        int
 	InstallTime time.Time
+	Encryptions int64
 }

vault/barrier_aes_gcm.go

@@ -31,6 +31,9 @@ const (
 
 	// termSize the number of bytes used for the key term.
 	termSize = 4
+
+	autoRotateCheckInterval = 5 * time.Minute
+	legacyRotateReason      = "legacy rotation"
 )
 
 // Versions of the AESGCM storage methodology
@@ -46,9 +49,11 @@ type barrierInit struct {
 }
 
 // Validate AESGCMBarrier satisfies SecurityBarrier interface
-var _ SecurityBarrier = &AESGCMBarrier{}
-
-var barrierEncryptsMetric = []string{"barrier", "estimated_encryptions"}
+var (
+	_                      SecurityBarrier = &AESGCMBarrier{}
+	barrierEncryptsMetric                  = []string{"barrier", "estimated_encryptions"}
+	barrierRotationsMetric                 = []string{"barrier", "auto_rotation"}
+)
 
 // AESGCMBarrier is a SecurityBarrier implementation that uses the AES
 // cipher core and the Galois Counter Mode block mode. It defaults to
@@ -76,6 +81,30 @@ type AESGCMBarrier struct {
 	currentAESGCMVersionByte byte
 
 	initialized atomic.Bool
+
+	UnaccountedEncryptions atomic.Int64
+	// Used only for testing
+	RemoteEncryptions     atomic.Int64
+	totalLocalEncryptions atomic.Int64
+}
+
+func (b *AESGCMBarrier) RotationConfig() (kc KeyRotationConfig, err error) {
+	if b.keyring == nil {
+		return kc, errors.New("keyring not yet present")
+	}
+	return b.keyring.rotationConfig, nil
+}
+
+func (b *AESGCMBarrier) SetRotationConfig(ctx context.Context, rotConfig KeyRotationConfig) error {
+	b.l.Lock()
+	defer b.l.Unlock()
+	rotConfig.Sanitize()
+	if !rotConfig.Equals(b.keyring.rotationConfig) {
+		b.keyring.rotationConfig = rotConfig
+
+		return b.persistKeyring(ctx, b.keyring)
+	}
+	return nil
 }
 
 // NewAESGCMBarrier is used to construct a new barrier that uses
@@ -222,7 +251,7 @@ func (b *AESGCMBarrier) persistKeyring(ctx context.Context, keyring *Keyring) er
 	if err != nil {
 		return err
 	}
-	value, err = b.encrypt(masterKeyPath, activeKey.Term, aead, keyBuf)
+	value, err = b.encryptTracked(masterKeyPath, activeKey.Term, aead, keyBuf)
 	if err != nil {
 		return err
 	}
@@ -315,8 +344,17 @@ func (b *AESGCMBarrier) ReloadKeyring(ctx context.Context) error {
 		return err
 	}
 
-	// Recover the keyring
-	keyring, err := DeserializeKeyring(plain)
+	// Reset enc. counters, this may be a leadership change
+	b.totalLocalEncryptions.Store(0)
+	b.totalLocalEncryptions.Store(0)
+	b.UnaccountedEncryptions.Store(0)
+	b.RemoteEncryptions.Store(0)
+
+	return b.recoverKeyring(plain)
+}
+
+func (b *AESGCMBarrier) recoverKeyring(plaintext []byte) error {
+	keyring, err := DeserializeKeyring(plaintext)
 	if err != nil {
 		return errwrap.Wrapf("keyring deserialization failed: {{err}}", err)
 	}
@@ -416,14 +454,13 @@ func (b *AESGCMBarrier) Unseal(ctx context.Context, key []byte) error {
 		}
 
 		// Recover the keyring
-		keyring, err := DeserializeKeyring(plain)
+		err = b.recoverKeyring(plain)
 		if err != nil {
 			return errwrap.Wrapf("keyring deserialization failed: {{err}}", err)
 		}
 
-		// Setup the keyring and finish
-		b.keyring = keyring
 		b.sealed = false
+
 		return nil
 	}
 
@@ -485,6 +522,7 @@ func (b *AESGCMBarrier) Unseal(ctx context.Context, key []byte) error {
 	// Set the vault as unsealed
 	b.keyring = keyring
 	b.sealed = false
+
 	return nil
 }
 
@@ -504,7 +542,7 @@ func (b *AESGCMBarrier) Seal() error {
 
 // Rotate is used to create a new encryption key. All future writes
 // should use the new key, while old values should still be decryptable.
-func (b *AESGCMBarrier) Rotate(ctx context.Context, reader io.Reader) (uint32, error) {
+func (b *AESGCMBarrier) Rotate(ctx context.Context, randomSource io.Reader) (uint32, error) {
 	b.l.Lock()
 	defer b.l.Unlock()
 	if b.sealed {
@@ -512,7 +550,7 @@ func (b *AESGCMBarrier) Rotate(ctx context.Context, reader io.Reader) (uint32, e
 	}
 
 	// Generate a new key
-	encrypt, err := b.GenerateKey(reader)
+	encrypt, err := b.GenerateKey(randomSource)
 	if err != nil {
 		return 0, errwrap.Wrapf("failed to generate encryption key: {{err}}", err)
 	}
@@ -536,8 +574,14 @@ func (b *AESGCMBarrier) Rotate(ctx context.Context, reader io.Reader) (uint32, e
 		return 0, err
 	}
 
+	// Clear encryption tracking
+	b.RemoteEncryptions.Store(0)
+	b.totalLocalEncryptions.Store(0)
+	b.UnaccountedEncryptions.Store(0)
+
 	// Swap the keyrings
 	b.keyring = newKeyring
+
 	return newTerm, nil
 }
 
@@ -567,7 +611,7 @@ func (b *AESGCMBarrier) CreateUpgrade(ctx context.Context, term uint32) error {
 	}
 
 	key := fmt.Sprintf("%s%d", keyringUpgradePrefix, prevTerm)
-	value, err := b.encrypt(key, prevTerm, primary, buf)
+	value, err := b.encryptTracked(key, prevTerm, primary, buf)
 	b.l.RUnlock()
 	if err != nil {
 		return err
@@ -668,6 +712,7 @@ func (b *AESGCMBarrier) ActiveKeyInfo() (*KeyInfo, error) {
 	info := &KeyInfo{
 		Term:        int(term),
 		InstallTime: key.InstallTime,
+		Encryptions: b.encryptions(),
 	}
 	return info, nil
 }
@@ -748,7 +793,7 @@ func (b *AESGCMBarrier) Put(ctx context.Context, entry *logical.StorageEntry) er
 }
 
 func (b *AESGCMBarrier) putInternal(ctx context.Context, term uint32, primary cipher.AEAD, entry *logical.StorageEntry) error {
-	value, err := b.encrypt(entry.Key, term, primary, entry.Value)
+	value, err := b.encryptTracked(entry.Key, term, primary, entry.Value)
 	if err != nil {
 		return err
 	}
@@ -935,8 +980,6 @@ func (b *AESGCMBarrier) encrypt(path string, term uint32, gcm cipher.AEAD, plain
 		return nil, errors.New("unable to read enough random bytes to fill gcm nonce")
 	}
 
-	metrics.IncrCounterWithLabels(barrierEncryptsMetric, 1, termLabel(term))
-
 	// Seal the output
 	switch b.currentAESGCMVersionByte {
 	case AESGCMVersion1:
@@ -986,7 +1029,7 @@ func (b *AESGCMBarrier) decrypt(path string, gcm cipher.AEAD, cipher []byte) ([]
 }
 
 // Encrypt is used to encrypt in-memory for the BarrierEncryptor interface
-func (b *AESGCMBarrier) Encrypt(_ context.Context, key string, plaintext []byte) ([]byte, error) {
+func (b *AESGCMBarrier) Encrypt(ctx context.Context, key string, plaintext []byte) ([]byte, error) {
 	b.l.RLock()
 	if b.sealed {
 		b.l.RUnlock()
@@ -1000,7 +1043,7 @@ func (b *AESGCMBarrier) Encrypt(_ context.Context, key string, plaintext []byte)
 		return nil, err
 	}
 
-	ciphertext, err := b.encrypt(key, term, primary, plaintext)
+	ciphertext, err := b.encryptTracked(key, term, primary, plaintext)
 	if err != nil {
 		return nil, err
 	}
@@ -1049,3 +1092,130 @@ func (b *AESGCMBarrier) Keyring() (*Keyring, error) {
 
 	return b.keyring.Clone(), nil
 }
+
+func (b *AESGCMBarrier) ConsumeEncryptionCount(consumer func(int64) error) error {
+	if b.keyring != nil {
+		// Lock to prevent replacement of the key while we consume the encryptions
+		b.l.RLock()
+		defer b.l.RUnlock()
+
+		c := b.UnaccountedEncryptions.Load()
+		err := consumer(c)
+		if err == nil && c > 0 {
+			// Consumer succeeded, remove those from local encryptions
+			b.UnaccountedEncryptions.Sub(c)
+		}
+		return err
+	}
+	return nil
+}
+
+func (b *AESGCMBarrier) AddRemoteEncryptions(encryptions int64) {
+	// For rollup and persistence
+	b.UnaccountedEncryptions.Add(encryptions)
+	// For testing
+	b.RemoteEncryptions.Add(encryptions)
+}
+
+func (b *AESGCMBarrier) encryptTracked(path string, term uint32, gcm cipher.AEAD, buf []byte) ([]byte, error) {
+	ct, err := b.encrypt(path, term, gcm, buf)
+	if err != nil {
+		return nil, err
+	}
+	// Increment the local encryption count, and track metrics
+	b.UnaccountedEncryptions.Add(1)
+	b.totalLocalEncryptions.Add(1)
+	metrics.IncrCounterWithLabels(barrierEncryptsMetric, 1, termLabel(term))
+
+	return ct, nil
+}
+
+// UnaccountedEncryptions returns the number of encryptions made on the local instance only for the current key term
+func (b *AESGCMBarrier) TotalLocalEncryptions() int64 {
+	return b.totalLocalEncryptions.Load()
+}
+
+func (b *AESGCMBarrier) CheckBarrierAutoRotate(ctx context.Context) (string, error) {
+	const oneYear = 24 * 365 * time.Hour
+	reason, err := func() (string, error) {
+		b.l.RLock()
+		defer b.l.RUnlock()
+		if b.keyring != nil {
+			// Rotation Checks
+			var reason string
+
+			rc, err := b.RotationConfig()
+			if err != nil {
+				b.l.RUnlock()
+				return "", err
+			}
+
+			if !rc.Disabled {
+				activeKey := b.keyring.ActiveKey()
+				ops := b.encryptions()
+				switch {
+				case activeKey.Encryptions == 0 && !activeKey.InstallTime.IsZero() && time.Since(activeKey.InstallTime) > oneYear:
+					reason = legacyRotateReason
+				case ops > b.keyring.rotationConfig.MaxOperations:
+					reason = "reached max operations"
+				case b.keyring.rotationConfig.Interval > 0 && time.Since(activeKey.InstallTime) > b.keyring.rotationConfig.Interval:
+					reason = "rotation interval reached"
+				}
+			}
+			return reason, nil
+		}
+		return "", nil
+	}()
+
+	if err != nil {
+		return "", err
+	}
+	if reason != "" {
+		return reason, nil
+	}
+
+	b.l.Lock()
+	defer b.l.Unlock()
+	if b.keyring != nil {
+		err := b.persistEncryptions(ctx)
+		if err != nil {
+			return "", err
+		}
+	}
+	return reason, nil
+}
+
+// Must be called with lock held
+func (b *AESGCMBarrier) persistEncryptions(ctx context.Context) error {
+	if !b.sealed {
+		// Encryption count persistence
+		upe := b.UnaccountedEncryptions.Load()
+		if upe > 0 {
+			activeKey := b.keyring.ActiveKey()
+			// Move local (unpersisted) encryptions to the key and persist.  This prevents us from needing to persist if
+			// there has been no activity. Since persistence performs an encryption, perversely we zero out after
+			// persistence and add 1 to the count to avoid this operation guaranteeing we need another
+			// autoRotateCheckInterval later.
+			newEncs := upe + 1
+			activeKey.Encryptions += uint64(newEncs)
+			newKeyring := b.keyring.Clone()
+			err := b.persistKeyring(ctx, newKeyring)
+			if err != nil {
+				return err
+			}
+			b.UnaccountedEncryptions.Sub(newEncs)
+		}
+	}
+	return nil
+}
+
+// Mostly for testing, returns the total number of encryption operations performed on the active term
+func (b *AESGCMBarrier) encryptions() int64 {
+	if b.keyring != nil {
+		activeKey := b.keyring.ActiveKey()
+		if activeKey != nil {
+			return b.UnaccountedEncryptions.Load() + int64(activeKey.Encryptions)
+		}
+	}
+	return 0
+}

vault/barrier_aes_gcm_test.go

@@ -6,6 +6,7 @@ import (
 	"crypto/rand"
 	"encoding/json"
 	"testing"
+	"time"
 
 	log "github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/vault/sdk/helper/logging"
@@ -594,5 +595,37 @@ func TestAESGCMBarrier_ReloadKeyring(t *testing.T) {
 	if len(b.cache) != 0 {
 		t.Fatal("failed to clear cache")
 	}
-
+}
+
+func TestBarrier_LegacyRotate(t *testing.T) {
+	inm, err := inmem.NewInmem(nil, logger)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	b1, err := NewAESGCMBarrier(inm)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	} // Initialize the barrier
+	key, _ := b1.GenerateKey(rand.Reader)
+	b1.Initialize(context.Background(), key, nil, rand.Reader)
+	err = b1.Unseal(context.Background(), key)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	k1 := b1.keyring.TermKey(1)
+	k1.Encryptions = 0
+	k1.InstallTime = time.Now().Add(-24 * 366 * time.Hour)
+	b1.persistKeyring(context.Background(), b1.keyring)
+	b1.Seal()
+
+	err = b1.Unseal(context.Background(), key)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	reason, err := b1.CheckBarrierAutoRotate(context.Background())
+	if err != nil || reason != legacyRotateReason {
+		t.Fail()
+	}
 }

vault/barrier_test.go

@@ -11,115 +11,7 @@ import (
 )
 
 func testBarrier(t *testing.T, b SecurityBarrier) {
-	// Should not be initialized
-	init, err := b.Initialized(context.Background())
-	if err != nil {
-		t.Fatalf("err: %v", err)
-	}
-	if init {
-		t.Fatalf("should not be initialized")
-	}
-
-	// Should start sealed
-	sealed, err := b.Sealed()
-	if err != nil {
-		t.Fatalf("err: %v", err)
-	}
-	if !sealed {
-		t.Fatalf("should be sealed")
-	}
-
-	// Sealing should be a no-op
-	if err := b.Seal(); err != nil {
-		t.Fatalf("err: %v", err)
-	}
-
-	// All operations should fail
-	e := &logical.StorageEntry{Key: "test", Value: []byte("test")}
-	if err := b.Put(context.Background(), e); err != ErrBarrierSealed {
-		t.Fatalf("err: %v", err)
-	}
-	if _, err := b.Get(context.Background(), "test"); err != ErrBarrierSealed {
-		t.Fatalf("err: %v", err)
-	}
-	if err := b.Delete(context.Background(), "test"); err != ErrBarrierSealed {
-		t.Fatalf("err: %v", err)
-	}
-	if _, err := b.List(context.Background(), ""); err != ErrBarrierSealed {
-		t.Fatalf("err: %v", err)
-	}
-
-	// Get a new key
-	key, err := b.GenerateKey(rand.Reader)
-	if err != nil {
-		t.Fatalf("err: %v", err)
-	}
-
-	// Validate minimum key length
-	min, max := b.KeyLength()
-	if min < 16 {
-		t.Fatalf("minimum key size too small: %d", min)
-	}
-	if max < min {
-		t.Fatalf("maximum key size smaller than min")
-	}
-
-	// Unseal should not work
-	if err := b.Unseal(context.Background(), key); err != ErrBarrierNotInit {
-		t.Fatalf("err: %v", err)
-	}
-
-	// Initialize the vault
-	if err := b.Initialize(context.Background(), key, nil, rand.Reader); err != nil {
-		t.Fatalf("err: %v", err)
-	}
-
-	// Double Initialize should fail
-	if err := b.Initialize(context.Background(), key, nil, rand.Reader); err != ErrBarrierAlreadyInit {
-		t.Fatalf("err: %v", err)
-	}
-
-	// Should be initialized
-	init, err = b.Initialized(context.Background())
-	if err != nil {
-		t.Fatalf("err: %v", err)
-	}
-	if !init {
-		t.Fatalf("should be initialized")
-	}
-
-	// Should still be sealed
-	sealed, err = b.Sealed()
-	if err != nil {
-		t.Fatalf("err: %v", err)
-	}
-	if !sealed {
-		t.Fatalf("should sealed")
-	}
-
-	// Unseal should work
-	if err := b.Unseal(context.Background(), key); err != nil {
-		t.Fatalf("err: %v", err)
-	}
-
-	// Unseal should no-op when done twice
-	if err := b.Unseal(context.Background(), key); err != nil {
-		t.Fatalf("err: %v", err)
-	}
-
-	// Should no longer be sealed
-	sealed, err = b.Sealed()
-	if err != nil {
-		t.Fatalf("err: %v", err)
-	}
-	if sealed {
-		t.Fatalf("should be unsealed")
-	}
-
-	// Verify the master key
-	if err := b.VerifyMaster(key); err != nil {
-		t.Fatalf("err: %v", err)
-	}
+	err, e, key := testInitAndUnseal(t, b)
 
 	// Operations should work
 	out, err := b.Get(context.Background(), "test")
@@ -244,6 +136,119 @@ func testBarrier(t *testing.T, b SecurityBarrier) {
 	}
 }
 
+func testInitAndUnseal(t *testing.T, b SecurityBarrier) (error, *logical.StorageEntry, []byte) {
+	// Should not be initialized
+	init, err := b.Initialized(context.Background())
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if init {
+		t.Fatalf("should not be initialized")
+	}
+
+	// Should start sealed
+	sealed, err := b.Sealed()
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if !sealed {
+		t.Fatalf("should be sealed")
+	}
+
+	// Sealing should be a no-op
+	if err := b.Seal(); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// All operations should fail
+	e := &logical.StorageEntry{Key: "test", Value: []byte("test")}
+	if err := b.Put(context.Background(), e); err != ErrBarrierSealed {
+		t.Fatalf("err: %v", err)
+	}
+	if _, err := b.Get(context.Background(), "test"); err != ErrBarrierSealed {
+		t.Fatalf("err: %v", err)
+	}
+	if err := b.Delete(context.Background(), "test"); err != ErrBarrierSealed {
+		t.Fatalf("err: %v", err)
+	}
+	if _, err := b.List(context.Background(), ""); err != ErrBarrierSealed {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Get a new key
+	key, err := b.GenerateKey(rand.Reader)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Validate minimum key length
+	min, max := b.KeyLength()
+	if min < 16 {
+		t.Fatalf("minimum key size too small: %d", min)
+	}
+	if max < min {
+		t.Fatalf("maximum key size smaller than min")
+	}
+
+	// Unseal should not work
+	if err := b.Unseal(context.Background(), key); err != ErrBarrierNotInit {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Initialize the vault
+	if err := b.Initialize(context.Background(), key, nil, rand.Reader); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Double Initialize should fail
+	if err := b.Initialize(context.Background(), key, nil, rand.Reader); err != ErrBarrierAlreadyInit {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Should be initialized
+	init, err = b.Initialized(context.Background())
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if !init {
+		t.Fatalf("should be initialized")
+	}
+
+	// Should still be sealed
+	sealed, err = b.Sealed()
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if !sealed {
+		t.Fatalf("should sealed")
+	}
+
+	// Unseal should work
+	if err := b.Unseal(context.Background(), key); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Unseal should no-op when done twice
+	if err := b.Unseal(context.Background(), key); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Should no longer be sealed
+	sealed, err = b.Sealed()
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if sealed {
+		t.Fatalf("should be unsealed")
+	}
+
+	// Verify the master key
+	if err := b.VerifyMaster(key); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	return err, e, key
+}
+
 func testBarrier_Rotate(t *testing.T, b SecurityBarrier) {
 	// Initialize the barrier
 	key, _ := b.GenerateKey(rand.Reader)

vault/core.go

@@ -556,6 +556,8 @@ type Core struct {
 	// for standby instances before we delete the upgrade keys
 	keyRotateGracePeriod *int64
 
+	autoRotateCancel context.CancelFunc
+
 	// number of workers to use for lease revocation in the expiration manager
 	numExpirationWorkers int
 
@@ -1761,6 +1763,11 @@ func (c *Core) sealInternalWithOptions(grabStateLock, keepHALock, performCleanup
 
 	c.logger.Info("marked as sealed")
 
+	// Give the barrier a chance to persist encryption counts
+	if c.autoRotateCancel != nil {
+		c.checkBarrierAutoRotate(c.activeContext)
+	}
+
 	// Clear forwarding clients
 	c.requestForwardingConnectionLock.Lock()
 	c.clearForwardingClients()
@@ -1904,6 +1911,13 @@ func (s standardUnsealStrategy) unseal(ctx context.Context, logger log.Logger, c
 		if err := c.persistFeatureFlags(ctx); err != nil {
 			return err
 		}
+
+	}
+
+	if c.autoRotateCancel == nil {
+		var autoRotateCtx context.Context
+		autoRotateCtx, c.autoRotateCancel = context.WithCancel(c.activeContext)
+		go c.autoRotateBarrierLoop(autoRotateCtx)
 	}
 
 	if !c.IsDRSecondary() {
@@ -2134,6 +2148,11 @@ func (c *Core) preSeal() error {
 		result = multierror.Append(result, err)
 	}
 
+	if c.autoRotateCancel != nil {
+		c.autoRotateCancel()
+		c.autoRotateCancel = nil
+	}
+
 	preSealPhysical(c)
 
 	c.logger.Info("pre-seal teardown complete")
@@ -2728,6 +2747,50 @@ func (c *Core) SetKeyRotateGracePeriod(t time.Duration) {
 	atomic.StoreInt64(c.keyRotateGracePeriod, int64(t))
 }
 
+// Periodically test whether to automatically rotate the barrier key
+func (c *Core) autoRotateBarrierLoop(ctx context.Context) {
+	t := time.NewTicker(autoRotateCheckInterval)
+	for {
+		select {
+		case <-t.C:
+			c.checkBarrierAutoRotate(ctx)
+		case <-ctx.Done():
+			t.Stop()
+			return
+		}
+	}
+}
+
+func (c *Core) checkBarrierAutoRotate(ctx context.Context) {
+	if c.isPrimary() {
+		reason, err := c.barrier.CheckBarrierAutoRotate(ctx)
+		if err != nil {
+			lf := c.logger.Error
+			if strings.HasSuffix(err.Error(), "context canceled") {
+				lf = c.logger.Debug
+			}
+			lf("error in barrier auto rotation", "error", err)
+			return
+		}
+		if reason != "" {
+			// Time to rotate.  Invoke the rotation handler in order to both rotate and create
+			// the replication canary
+			c.logger.Info("automatic barrier key rotation triggered", "reason", reason)
+
+			_, err := c.systemBackend.handleRotate(ctx, nil, nil)
+			if err != nil {
+				c.logger.Error("error automatically rotating barrier key", "error", err)
+			} else {
+				metrics.IncrCounter(barrierRotationsMetric, 1)
+			}
+		}
+	}
+}
+
+func (c *Core) isPrimary() bool {
+	return !c.ReplicationState().HasState(consts.ReplicationPerformanceSecondary | consts.ReplicationDRSecondary)
+}
+
 func ParseRequiredState(raw string, hmacKey []byte) (*logical.WALState, error) {
 	cooked, err := base64.StdEncoding.DecodeString(raw)
 	if err != nil {

vault/core_metrics.go

@@ -10,6 +10,7 @@ import (
 	"github.com/armon/go-metrics"
 	"github.com/hashicorp/vault/helper/metricsutil"
 	"github.com/hashicorp/vault/helper/namespace"
+	"github.com/hashicorp/vault/sdk/helper/consts"
 	"github.com/hashicorp/vault/sdk/logical"
 )
 
@@ -69,8 +70,19 @@ func (c *Core) metricsLoop(stopCh chan struct{}) {
 				continue
 			}
 			if c.perfStandby { // already have lock here, do not re-acquire
-				syncCounter(c)
+				err := syncCounters(c)
+				if err != nil {
+					c.logger.Error("writing syncing counters", "err", err)
+				}
 			} else {
+				// Perf standbys will have synced above, but active nodes on a secondary cluster still need to ship
+				// barrier encryption counts
+				if c.ReplicationState().HasState(consts.ReplicationPerformanceSecondary) {
+					err := syncBarrierEncryptionCounter(c)
+					if err != nil {
+						c.logger.Error("writing syncing encryption counts", "err", err)
+					}
+				}
 				err := c.saveCurrentRequestCounters(context.Background(), time.Now())
 				if err != nil {
 					c.logger.Error("writing request counters to barrier", "err", err)

vault/keyring.go

@@ -4,13 +4,29 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"sync/atomic"
 	"time"
 
 	"github.com/hashicorp/errwrap"
 	"github.com/hashicorp/vault/sdk/helper/jsonutil"
 )
 
+const (
+	// 10% shy of the NIST recommended maximum, leaving a buffer to account for
+	// tracking losses.
+	absoluteOperationMaximum = int64(3_865_470_566)
+	absoluteOperationMinimum = int64(1_000_000)
+	minimumRotationInterval  = 24 * time.Hour
+)
+
+var (
+	defaultRotationConfig = KeyRotationConfig{
+		MaxOperations: absoluteOperationMaximum,
+	}
+	disabledRotationConfig = KeyRotationConfig{
+		Disabled: true,
+	}
+)
+
 // Keyring is used to manage multiple encryption keys used by
 // the barrier. New keys can be installed and each has a sequential term.
 // The term used to encrypt a key is prefixed to the key written out.
@@ -20,25 +36,32 @@ import (
 // when a new key is added to the keyring, we can encrypt with the master key
 // and write out the new keyring.
 type Keyring struct {
-	masterKey  []byte
-	keys       map[uint32]*Key
-	activeTerm uint32
+	masterKey      []byte
+	keys           map[uint32]*Key
+	activeTerm     uint32
+	rotationConfig KeyRotationConfig
 }
 
 // EncodedKeyring is used for serialization of the keyring
 type EncodedKeyring struct {
-	MasterKey []byte
-	Keys      []*Key
+	MasterKey      []byte
+	Keys           []*Key
+	RotationConfig KeyRotationConfig
 }
 
 // Key represents a single term, along with the key used.
 type Key struct {
-	Term                uint32
-	Version             int
-	Value               []byte
-	InstallTime         time.Time
-	Encryptions         uint64
-	ReportedEncryptions uint64 `json:",omitempty"`
+	Term        uint32
+	Version     int
+	Value       []byte
+	InstallTime time.Time
+	Encryptions uint64 `json:"encryptions,omitempty"`
+}
+
+type KeyRotationConfig struct {
+	Disabled      bool
+	MaxOperations int64
+	Interval      time.Duration
 }
 
 // Serialize is used to create a byte encoded key
@@ -58,8 +81,9 @@ func DeserializeKey(buf []byte) (*Key, error) {
 // NewKeyring creates a new keyring
 func NewKeyring() *Keyring {
 	k := &Keyring{
-		keys:       make(map[uint32]*Key),
-		activeTerm: 0,
+		keys:           make(map[uint32]*Key),
+		activeTerm:     0,
+		rotationConfig: defaultRotationConfig,
 	}
 	return k
 }
@@ -67,9 +91,10 @@ func NewKeyring() *Keyring {
 // Clone returns a new copy of the keyring
 func (k *Keyring) Clone() *Keyring {
 	clone := &Keyring{
-		masterKey:  k.masterKey,
-		keys:       make(map[uint32]*Key, len(k.keys)),
-		activeTerm: k.activeTerm,
+		masterKey:      k.masterKey,
+		keys:           make(map[uint32]*Key, len(k.keys)),
+		activeTerm:     k.activeTerm,
+		rotationConfig: k.rotationConfig,
 	}
 	for idx, key := range k.keys {
 		clone.keys[idx] = key
@@ -102,6 +127,14 @@ func (k *Keyring) AddKey(key *Key) (*Keyring, error) {
 	if key.Term > clone.activeTerm {
 		clone.activeTerm = key.Term
 	}
+
+	// Zero out encryption estimates for previous terms
+	for term, key := range clone.keys {
+		if term != clone.activeTerm {
+			key.Encryptions = 0
+		}
+	}
+
 	return clone, nil
 }
 
@@ -156,7 +189,8 @@ func (k *Keyring) MasterKey() []byte {
 func (k *Keyring) Serialize() ([]byte, error) {
 	// Create the encoded entry
 	enc := EncodedKeyring{
-		MasterKey: k.masterKey,
+		MasterKey:      k.masterKey,
+		RotationConfig: k.rotationConfig,
 	}
 	for _, key := range k.keys {
 		enc.Keys = append(enc.Keys, key)
@@ -205,9 +239,15 @@ func (k *Keyring) Zeroize(keysToo bool) {
 	}
 }
 
-func (k *Keyring) AddEncryptionEstimate(term uint32, delta uint64) {
-	key := k.TermKey(term)
-	if key != nil {
-		atomic.AddUint64(&key.Encryptions, delta)
+func (c *KeyRotationConfig) Sanitize() {
+	if c.MaxOperations == 0 || c.MaxOperations > absoluteOperationMaximum || c.MaxOperations < absoluteOperationMinimum {
+		c.MaxOperations = absoluteOperationMaximum
+	}
+	if c.Interval > 0 && c.Interval < minimumRotationInterval {
+		c.Interval = minimumRotationInterval
 	}
 }
+
+func (c *KeyRotationConfig) Equals(config KeyRotationConfig) bool {
+	return c.MaxOperations == config.MaxOperations && c.Interval == config.Interval
+}

vault/logical_system.go

@@ -2525,52 +2525,82 @@ func (b *SystemBackend) handleKeyStatus(ctx context.Context, req *logical.Reques
 		Data: map[string]interface{}{
 			"term":         info.Term,
 			"install_time": info.InstallTime.Format(time.RFC3339Nano),
+			"encryptions":  info.Encryptions,
 		},
 	}
 	return resp, nil
 }
 
+// handleKeyRotationConfigRead returns the barrier key rotation config
+func (b *SystemBackend) handleKeyRotationConfigRead(_ context.Context, _ *logical.Request, _ *framework.FieldData) (*logical.Response, error) {
+	// Get the key info
+	rotConfig, err := b.Core.barrier.RotationConfig()
+	if err != nil {
+		return nil, err
+	}
+
+	resp := &logical.Response{
+		Data: map[string]interface{}{
+			"max_operations": rotConfig.MaxOperations,
+			"enabled":        !rotConfig.Disabled,
+		},
+	}
+	if rotConfig.Interval > 0 {
+		resp.Data["interval"] = rotConfig.Interval.String()
+	} else {
+		resp.Data["interval"] = 0
+	}
+	return resp, nil
+}
+
+// handleKeyRotationConfigRead returns the barrier key rotation config
+func (b *SystemBackend) handleKeyRotationConfigUpdate(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
+	rotConfig, err := b.Core.barrier.RotationConfig()
+	if err != nil {
+		return nil, err
+	}
+	maxOps, ok, err := data.GetOkErr("max_operations")
+	if err != nil {
+		return nil, err
+	}
+	if ok {
+		rotConfig.MaxOperations = int64(maxOps.(int))
+	}
+	interval, ok, err := data.GetOkErr("interval")
+	if err != nil {
+		return nil, err
+	}
+	if ok {
+		rotConfig.Interval = time.Second * time.Duration(interval.(int))
+	}
+
+	enabled, ok, err := data.GetOkErr("enabled")
+	if err != nil {
+		return nil, err
+	}
+	if ok {
+		rotConfig.Disabled = !enabled.(bool)
+	}
+	// Store the rotation config
+	b.Core.barrier.SetRotationConfig(ctx, rotConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	return nil, nil
+}
+
 // handleRotate is used to trigger a key rotation
-func (b *SystemBackend) handleRotate(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
+func (b *SystemBackend) handleRotate(ctx context.Context, _ *logical.Request, _ *framework.FieldData) (*logical.Response, error) {
 	repState := b.Core.ReplicationState()
 	if repState.HasState(consts.ReplicationPerformanceSecondary) {
 		return logical.ErrorResponse("cannot rotate on a replication secondary"), nil
 	}
 
-	// Rotate to the new term
-	newTerm, err := b.Core.barrier.Rotate(ctx, b.Core.secureRandomReader)
-	if err != nil {
-		b.Backend.Logger().Error("failed to create new encryption key", "error", err)
+	if err := b.rotateBarrierKey(ctx); err != nil {
+		b.Backend.Logger().Error("error handling key rotation", "error", err)
 		return handleError(err)
 	}
-	b.Backend.Logger().Info("installed new encryption key")
-
-	// In HA mode, we need to an upgrade path for the standby instances
-	if b.Core.ha != nil {
-		// Create the upgrade path to the new term
-		if err := b.Core.barrier.CreateUpgrade(ctx, newTerm); err != nil {
-			b.Backend.Logger().Error("failed to create new upgrade", "term", newTerm, "error", err)
-		}
-
-		// Schedule the destroy of the upgrade path
-		time.AfterFunc(b.Core.KeyRotateGracePeriod(), func() {
-			b.Backend.Logger().Debug("cleaning up upgrade keys", "waited", b.Core.KeyRotateGracePeriod())
-			if err := b.Core.barrier.DestroyUpgrade(b.Core.activeContext, newTerm); err != nil {
-				b.Backend.Logger().Error("failed to destroy upgrade", "term", newTerm, "error", err)
-			}
-		})
-	}
-
-	// Write to the canary path, which will force a synchronous truing during
-	// replication
-	if err := b.Core.barrier.Put(ctx, &logical.StorageEntry{
-		Key:   coreKeyringCanaryPath,
-		Value: []byte(fmt.Sprintf("new-rotation-term-%d", newTerm)),
-	}); err != nil {
-		b.Core.logger.Error("error saving keyring canary", "error", err)
-		return nil, errwrap.Wrapf("failed to save keyring canary: {{err}}", err)
-	}
-
 	return nil, nil
 }
 
@@ -3842,6 +3872,44 @@ func (b *SystemBackend) verifyDROperationTokenOnSecondary(f framework.OperationF
 	return f
 }
 
+func (b *SystemBackend) rotateBarrierKey(ctx context.Context) error {
+	// Rotate to the new term
+	newTerm, err := b.Core.barrier.Rotate(ctx, b.Core.secureRandomReader)
+	if err != nil {
+		return errwrap.Wrap(errors.New("failed to create new encryption key"), err)
+	}
+	b.Backend.Logger().Info("installed new encryption key")
+
+	// In HA mode, we need to an upgrade path for the standby instances
+	if b.Core.ha != nil {
+		// Create the upgrade path to the new term
+		if err := b.Core.barrier.CreateUpgrade(ctx, newTerm); err != nil {
+			b.Backend.Logger().Error("failed to create new upgrade", "term", newTerm, "error", err)
+		}
+
+		// Schedule the destroy of the upgrade path
+		time.AfterFunc(b.Core.KeyRotateGracePeriod(), func() {
+			b.Backend.Logger().Debug("cleaning up upgrade keys", "waited", b.Core.KeyRotateGracePeriod())
+			if err := b.Core.barrier.DestroyUpgrade(b.Core.activeContext, newTerm); err != nil {
+				b.Backend.Logger().Error("failed to destroy upgrade", "term", newTerm, "error", err)
+			}
+		})
+	}
+
+	// Write to the canary path, which will force a synchronous truing during
+	// replication
+	if err := b.Core.barrier.Put(ctx, &logical.StorageEntry{
+		Key:   coreKeyringCanaryPath,
+		Value: []byte(fmt.Sprintf("new-rotation-term-%d", newTerm)),
+	}); err != nil {
+		b.Core.logger.Error("error saving keyring canary", "error", err)
+		return errwrap.Wrap(errors.New("failed to save keyring canary"), err)
+	}
+
+	return nil
+
+}
+
 func sanitizePath(path string) string {
 	if !strings.HasSuffix(path, "/") {
 		path += "/"
@@ -4339,6 +4407,25 @@ Enable a new audit backend or disable an existing backend.
 		`,
 	},
 
+	"rotate-config": {
+		"Configures settings related to the backend encryption key management.",
+		`
+		Configures settings related to the automatic rotation of the backend encryption key.
+		`,
+	},
+
+	"rotation-enabled": {
+		"Whether automatic rotation is enabled.",
+		"",
+	},
+	"rotation-max-operations": {
+		"The number of encryption operations performed before the barrier key is automatically rotated.",
+		"",
+	},
+	"rotation-interval": {
+		"How long after installation of an active key term that the key will be automatically rotated.",
+		"",
+	},
 	"rotate": {
 		"Rotates the backend encryption key used to persist data.",
 		`

vault/logical_system_paths.go

@@ -602,6 +602,31 @@ func (b *SystemBackend) sealPaths() []*framework.Path {
 			HelpDescription: strings.TrimSpace(sysHelp["key-status"][1]),
 		},
 
+		{
+			Pattern: "rotate/config$",
+			Fields: map[string]*framework.FieldSchema{
+				"enabled": &framework.FieldSchema{
+					Type:        framework.TypeBool,
+					Description: strings.TrimSpace(sysHelp["rotation-enabled"][0]),
+				},
+				"max_operations": &framework.FieldSchema{
+					Type:        framework.TypeInt, //64?
+					Description: strings.TrimSpace(sysHelp["rotation-max-operations"][0]),
+				},
+				"interval": &framework.FieldSchema{
+					Type:        framework.TypeDurationSecond,
+					Description: strings.TrimSpace(sysHelp["rotation-interval"][0]),
+				},
+			},
+			Callbacks: map[logical.Operation]framework.OperationFunc{
+				logical.ReadOperation:   b.handleKeyRotationConfigRead,
+				logical.UpdateOperation: b.handleKeyRotationConfigUpdate,
+			},
+
+			HelpSynopsis:    strings.TrimSpace(sysHelp["rotate-config"][0]),
+			HelpDescription: strings.TrimSpace(sysHelp["rotate-config"][1]),
+		},
+
 		{
 			Pattern: "rotate$",
 

vault/logical_system_test.go

@@ -2044,6 +2044,50 @@ func TestSystemBackend_keyStatus(t *testing.T) {
 		"term": 1,
 	}
 	delete(resp.Data, "install_time")
+	delete(resp.Data, "encryptions")
+	if !reflect.DeepEqual(resp.Data, exp) {
+		t.Fatalf("got: %#v expect: %#v", resp.Data, exp)
+	}
+}
+
+func TestSystemBackend_rotateConfig(t *testing.T) {
+	b := testSystemBackend(t)
+	req := logical.TestRequest(t, logical.ReadOperation, "rotate/config")
+	resp, err := b.HandleRequest(namespace.RootContext(nil), req)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	exp := map[string]interface{}{
+		"max_operations": absoluteOperationMaximum,
+		"interval":       0,
+		"enabled":        true,
+	}
+	if !reflect.DeepEqual(resp.Data, exp) {
+		t.Fatalf("got: %#v expect: %#v", resp.Data, exp)
+	}
+
+	req2 := logical.TestRequest(t, logical.UpdateOperation, "rotate/config")
+	req2.Data["max_operations"] = 2345678910
+	req2.Data["interval"] = "5432h0m0s"
+	req2.Data["enabled"] = false
+
+	resp, err = b.HandleRequest(namespace.RootContext(nil), req2)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	resp, err = b.HandleRequest(namespace.RootContext(nil), req)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	exp = map[string]interface{}{
+		"max_operations": int64(2345678910),
+		"interval":       "5432h0m0s",
+		"enabled":        false,
+	}
+
 	if !reflect.DeepEqual(resp.Data, exp) {
 		t.Fatalf("got: %#v expect: %#v", resp.Data, exp)
 	}
@@ -2071,6 +2115,7 @@ func TestSystemBackend_rotate(t *testing.T) {
 		"term": 2,
 	}
 	delete(resp.Data, "install_time")
+	delete(resp.Data, "encryptions")
 	if !reflect.DeepEqual(resp.Data, exp) {
 		t.Fatalf("got: %#v expect: %#v", resp.Data, exp)
 	}

vault/request_handling_util.go

@@ -26,7 +26,12 @@ func shouldForward(c *Core, resp *logical.Response, err error) bool {
 	return false
 }
 
-func syncCounter(c *Core) {
+func syncCounters(c *Core) error {
+	return nil
+}
+
+func syncBarrierEncryptionCounter(c *Core) error {
+	return nil
 }
 
 func couldForward(c *Core) bool {

vendor/github.com/hashicorp/vault/api/sys_rotate.go

@@ -68,10 +68,24 @@ func (c *Sys) KeyStatus() (*KeyStatus, error) {
 	}
 	result.InstallTime = installTime
 
+	encryptionsRaw, ok := secret.Data["encryptions"]
+	if ok {
+		encryptions, ok := encryptionsRaw.(json.Number)
+		if !ok {
+			return nil, errors.New("could not convert encryptions to a number")
+		}
+		encryptions64, err := encryptions.Int64()
+		if err != nil {
+			return nil, err
+		}
+		result.Encryptions = int(encryptions64)
+	}
+
 	return &result, err
 }
 
 type KeyStatus struct {
 	Term        int       `json:"term"`
 	InstallTime time.Time `json:"install_time"`
+	Encryptions int       `json:"encryptions"`
 }

vendor/github.com/hashicorp/vault/sdk/physical/file/file.go

@@ -225,9 +225,14 @@ func (b *FileBackend) PutInternal(ctx context.Context, entry *physical.Entry) er
 	if err := b.validatePath(entry.Key); err != nil {
 		return err
 	}
-
 	path, key := b.expandPath(entry.Key)
 
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	default:
+	}
+
 	// Make the parent tree
 	if err := os.MkdirAll(path, 0700); err != nil {
 		return err
@@ -249,12 +254,6 @@ func (b *FileBackend) PutInternal(ctx context.Context, entry *physical.Entry) er
 		return errors.New("could not successfully get a file handle")
 	}
 
-	select {
-	case <-ctx.Done():
-		return ctx.Err()
-	default:
-	}
-
 	enc := json.NewEncoder(f)
 	encErr := enc.Encode(&fileEntry{
 		Value: entry.Value,