diff --git a/api/sys_rotate.go b/api/sys_rotate.go index c525feb00d..e081587b11 100644 --- a/api/sys_rotate.go +++ b/api/sys_rotate.go @@ -68,10 +68,24 @@ func (c *Sys) KeyStatus() (*KeyStatus, error) { } result.InstallTime = installTime + encryptionsRaw, ok := secret.Data["encryptions"] + if ok { + encryptions, ok := encryptionsRaw.(json.Number) + if !ok { + return nil, errors.New("could not convert encryptions to a number") + } + encryptions64, err := encryptions.Int64() + if err != nil { + return nil, err + } + result.Encryptions = int(encryptions64) + } + return &result, err } type KeyStatus struct { Term int `json:"term"` InstallTime time.Time `json:"install_time"` + Encryptions int `json:"encryptions"` } diff --git a/http/sys_rotate_test.go b/http/sys_rotate_test.go index 7691ca042c..81597c7008 100644 --- a/http/sys_rotate_test.go +++ b/http/sys_rotate_test.go @@ -36,15 +36,16 @@ func TestSysRotate(t *testing.T) { testResponseStatus(t, resp, 200) testResponseBody(t, resp, &actual) - actualInstallTime, ok := actual["data"].(map[string]interface{})["install_time"] - if !ok || actualInstallTime == "" { - t.Fatal("install_time missing in data") + for _, field := range []string{"install_time", "encryptions"} { + actualVal, ok := actual["data"].(map[string]interface{})[field] + if !ok || actualVal == "" { + t.Fatal(field, " missing in data") + } + expected["data"].(map[string]interface{})[field] = actualVal + expected[field] = actualVal } - expected["data"].(map[string]interface{})["install_time"] = actualInstallTime - expected["install_time"] = actualInstallTime expected["request_id"] = actual["request_id"] - if diff := deep.Equal(actual, expected); diff != nil { t.Fatal(diff) } diff --git a/sdk/physical/file/file.go b/sdk/physical/file/file.go index 736ec41597..d08d1c2b67 100644 --- a/sdk/physical/file/file.go +++ b/sdk/physical/file/file.go @@ -225,9 +225,14 @@ func (b *FileBackend) PutInternal(ctx context.Context, entry *physical.Entry) er if err := b.validatePath(entry.Key); err != nil { return err } - path, key := b.expandPath(entry.Key) + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + // Make the parent tree if err := os.MkdirAll(path, 0700); err != nil { return err @@ -249,12 +254,6 @@ func (b *FileBackend) PutInternal(ctx context.Context, entry *physical.Entry) er return errors.New("could not successfully get a file handle") } - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - enc := json.NewEncoder(f) encErr := enc.Encode(&fileEntry{ Value: entry.Value, diff --git a/vault/barrier.go b/vault/barrier.go index dc0b2698cd..eaa3eb2e76 100644 --- a/vault/barrier.go +++ b/vault/barrier.go @@ -136,12 +136,28 @@ type SecurityBarrier interface { // ActiveKeyInfo is used to inform details about the active key ActiveKeyInfo() (*KeyInfo, error) + // RotationConfig returns the auto-rotation config for the barrier key + RotationConfig() (KeyRotationConfig, error) + + // SetRotationConfig updates the auto-rotation config for the barrier key + SetRotationConfig(ctx context.Context, config KeyRotationConfig) error + // Rekey is used to change the master key used to protect the keyring Rekey(context.Context, []byte) error // For replication we must send over the keyring, so this must be available Keyring() (*Keyring, error) + // For encryption count shipping, a function which handles updating local encryption counts if the consumer succeeds. + // This isolates the barrier code from the replication system + ConsumeEncryptionCount(consumer func(int64) error) error + + // Add encryption counts from a remote source (downstream cluster node) + AddRemoteEncryptions(encryptions int64) + + // Check whether an automatic rotation is due + CheckBarrierAutoRotate(ctx context.Context) (string, error) + // SecurityBarrier must provide the storage APIs logical.Storage @@ -177,4 +193,5 @@ type BarrierEncryptor interface { type KeyInfo struct { Term int InstallTime time.Time + Encryptions int64 } diff --git a/vault/barrier_aes_gcm.go b/vault/barrier_aes_gcm.go index 1a89e3f523..dd5a38159b 100644 --- a/vault/barrier_aes_gcm.go +++ b/vault/barrier_aes_gcm.go @@ -31,6 +31,9 @@ const ( // termSize the number of bytes used for the key term. termSize = 4 + + autoRotateCheckInterval = 5 * time.Minute + legacyRotateReason = "legacy rotation" ) // Versions of the AESGCM storage methodology @@ -46,9 +49,11 @@ type barrierInit struct { } // Validate AESGCMBarrier satisfies SecurityBarrier interface -var _ SecurityBarrier = &AESGCMBarrier{} - -var barrierEncryptsMetric = []string{"barrier", "estimated_encryptions"} +var ( + _ SecurityBarrier = &AESGCMBarrier{} + barrierEncryptsMetric = []string{"barrier", "estimated_encryptions"} + barrierRotationsMetric = []string{"barrier", "auto_rotation"} +) // AESGCMBarrier is a SecurityBarrier implementation that uses the AES // cipher core and the Galois Counter Mode block mode. It defaults to @@ -76,6 +81,30 @@ type AESGCMBarrier struct { currentAESGCMVersionByte byte initialized atomic.Bool + + UnaccountedEncryptions atomic.Int64 + // Used only for testing + RemoteEncryptions atomic.Int64 + totalLocalEncryptions atomic.Int64 +} + +func (b *AESGCMBarrier) RotationConfig() (kc KeyRotationConfig, err error) { + if b.keyring == nil { + return kc, errors.New("keyring not yet present") + } + return b.keyring.rotationConfig, nil +} + +func (b *AESGCMBarrier) SetRotationConfig(ctx context.Context, rotConfig KeyRotationConfig) error { + b.l.Lock() + defer b.l.Unlock() + rotConfig.Sanitize() + if !rotConfig.Equals(b.keyring.rotationConfig) { + b.keyring.rotationConfig = rotConfig + + return b.persistKeyring(ctx, b.keyring) + } + return nil } // NewAESGCMBarrier is used to construct a new barrier that uses @@ -222,7 +251,7 @@ func (b *AESGCMBarrier) persistKeyring(ctx context.Context, keyring *Keyring) er if err != nil { return err } - value, err = b.encrypt(masterKeyPath, activeKey.Term, aead, keyBuf) + value, err = b.encryptTracked(masterKeyPath, activeKey.Term, aead, keyBuf) if err != nil { return err } @@ -315,8 +344,17 @@ func (b *AESGCMBarrier) ReloadKeyring(ctx context.Context) error { return err } - // Recover the keyring - keyring, err := DeserializeKeyring(plain) + // Reset enc. counters, this may be a leadership change + b.totalLocalEncryptions.Store(0) + b.totalLocalEncryptions.Store(0) + b.UnaccountedEncryptions.Store(0) + b.RemoteEncryptions.Store(0) + + return b.recoverKeyring(plain) +} + +func (b *AESGCMBarrier) recoverKeyring(plaintext []byte) error { + keyring, err := DeserializeKeyring(plaintext) if err != nil { return errwrap.Wrapf("keyring deserialization failed: {{err}}", err) } @@ -416,14 +454,13 @@ func (b *AESGCMBarrier) Unseal(ctx context.Context, key []byte) error { } // Recover the keyring - keyring, err := DeserializeKeyring(plain) + err = b.recoverKeyring(plain) if err != nil { return errwrap.Wrapf("keyring deserialization failed: {{err}}", err) } - // Setup the keyring and finish - b.keyring = keyring b.sealed = false + return nil } @@ -485,6 +522,7 @@ func (b *AESGCMBarrier) Unseal(ctx context.Context, key []byte) error { // Set the vault as unsealed b.keyring = keyring b.sealed = false + return nil } @@ -504,7 +542,7 @@ func (b *AESGCMBarrier) Seal() error { // Rotate is used to create a new encryption key. All future writes // should use the new key, while old values should still be decryptable. -func (b *AESGCMBarrier) Rotate(ctx context.Context, reader io.Reader) (uint32, error) { +func (b *AESGCMBarrier) Rotate(ctx context.Context, randomSource io.Reader) (uint32, error) { b.l.Lock() defer b.l.Unlock() if b.sealed { @@ -512,7 +550,7 @@ func (b *AESGCMBarrier) Rotate(ctx context.Context, reader io.Reader) (uint32, e } // Generate a new key - encrypt, err := b.GenerateKey(reader) + encrypt, err := b.GenerateKey(randomSource) if err != nil { return 0, errwrap.Wrapf("failed to generate encryption key: {{err}}", err) } @@ -536,8 +574,14 @@ func (b *AESGCMBarrier) Rotate(ctx context.Context, reader io.Reader) (uint32, e return 0, err } + // Clear encryption tracking + b.RemoteEncryptions.Store(0) + b.totalLocalEncryptions.Store(0) + b.UnaccountedEncryptions.Store(0) + // Swap the keyrings b.keyring = newKeyring + return newTerm, nil } @@ -567,7 +611,7 @@ func (b *AESGCMBarrier) CreateUpgrade(ctx context.Context, term uint32) error { } key := fmt.Sprintf("%s%d", keyringUpgradePrefix, prevTerm) - value, err := b.encrypt(key, prevTerm, primary, buf) + value, err := b.encryptTracked(key, prevTerm, primary, buf) b.l.RUnlock() if err != nil { return err @@ -668,6 +712,7 @@ func (b *AESGCMBarrier) ActiveKeyInfo() (*KeyInfo, error) { info := &KeyInfo{ Term: int(term), InstallTime: key.InstallTime, + Encryptions: b.encryptions(), } return info, nil } @@ -748,7 +793,7 @@ func (b *AESGCMBarrier) Put(ctx context.Context, entry *logical.StorageEntry) er } func (b *AESGCMBarrier) putInternal(ctx context.Context, term uint32, primary cipher.AEAD, entry *logical.StorageEntry) error { - value, err := b.encrypt(entry.Key, term, primary, entry.Value) + value, err := b.encryptTracked(entry.Key, term, primary, entry.Value) if err != nil { return err } @@ -935,8 +980,6 @@ func (b *AESGCMBarrier) encrypt(path string, term uint32, gcm cipher.AEAD, plain return nil, errors.New("unable to read enough random bytes to fill gcm nonce") } - metrics.IncrCounterWithLabels(barrierEncryptsMetric, 1, termLabel(term)) - // Seal the output switch b.currentAESGCMVersionByte { case AESGCMVersion1: @@ -986,7 +1029,7 @@ func (b *AESGCMBarrier) decrypt(path string, gcm cipher.AEAD, cipher []byte) ([] } // Encrypt is used to encrypt in-memory for the BarrierEncryptor interface -func (b *AESGCMBarrier) Encrypt(_ context.Context, key string, plaintext []byte) ([]byte, error) { +func (b *AESGCMBarrier) Encrypt(ctx context.Context, key string, plaintext []byte) ([]byte, error) { b.l.RLock() if b.sealed { b.l.RUnlock() @@ -1000,7 +1043,7 @@ func (b *AESGCMBarrier) Encrypt(_ context.Context, key string, plaintext []byte) return nil, err } - ciphertext, err := b.encrypt(key, term, primary, plaintext) + ciphertext, err := b.encryptTracked(key, term, primary, plaintext) if err != nil { return nil, err } @@ -1049,3 +1092,130 @@ func (b *AESGCMBarrier) Keyring() (*Keyring, error) { return b.keyring.Clone(), nil } + +func (b *AESGCMBarrier) ConsumeEncryptionCount(consumer func(int64) error) error { + if b.keyring != nil { + // Lock to prevent replacement of the key while we consume the encryptions + b.l.RLock() + defer b.l.RUnlock() + + c := b.UnaccountedEncryptions.Load() + err := consumer(c) + if err == nil && c > 0 { + // Consumer succeeded, remove those from local encryptions + b.UnaccountedEncryptions.Sub(c) + } + return err + } + return nil +} + +func (b *AESGCMBarrier) AddRemoteEncryptions(encryptions int64) { + // For rollup and persistence + b.UnaccountedEncryptions.Add(encryptions) + // For testing + b.RemoteEncryptions.Add(encryptions) +} + +func (b *AESGCMBarrier) encryptTracked(path string, term uint32, gcm cipher.AEAD, buf []byte) ([]byte, error) { + ct, err := b.encrypt(path, term, gcm, buf) + if err != nil { + return nil, err + } + // Increment the local encryption count, and track metrics + b.UnaccountedEncryptions.Add(1) + b.totalLocalEncryptions.Add(1) + metrics.IncrCounterWithLabels(barrierEncryptsMetric, 1, termLabel(term)) + + return ct, nil +} + +// UnaccountedEncryptions returns the number of encryptions made on the local instance only for the current key term +func (b *AESGCMBarrier) TotalLocalEncryptions() int64 { + return b.totalLocalEncryptions.Load() +} + +func (b *AESGCMBarrier) CheckBarrierAutoRotate(ctx context.Context) (string, error) { + const oneYear = 24 * 365 * time.Hour + reason, err := func() (string, error) { + b.l.RLock() + defer b.l.RUnlock() + if b.keyring != nil { + // Rotation Checks + var reason string + + rc, err := b.RotationConfig() + if err != nil { + b.l.RUnlock() + return "", err + } + + if !rc.Disabled { + activeKey := b.keyring.ActiveKey() + ops := b.encryptions() + switch { + case activeKey.Encryptions == 0 && !activeKey.InstallTime.IsZero() && time.Since(activeKey.InstallTime) > oneYear: + reason = legacyRotateReason + case ops > b.keyring.rotationConfig.MaxOperations: + reason = "reached max operations" + case b.keyring.rotationConfig.Interval > 0 && time.Since(activeKey.InstallTime) > b.keyring.rotationConfig.Interval: + reason = "rotation interval reached" + } + } + return reason, nil + } + return "", nil + }() + + if err != nil { + return "", err + } + if reason != "" { + return reason, nil + } + + b.l.Lock() + defer b.l.Unlock() + if b.keyring != nil { + err := b.persistEncryptions(ctx) + if err != nil { + return "", err + } + } + return reason, nil +} + +// Must be called with lock held +func (b *AESGCMBarrier) persistEncryptions(ctx context.Context) error { + if !b.sealed { + // Encryption count persistence + upe := b.UnaccountedEncryptions.Load() + if upe > 0 { + activeKey := b.keyring.ActiveKey() + // Move local (unpersisted) encryptions to the key and persist. This prevents us from needing to persist if + // there has been no activity. Since persistence performs an encryption, perversely we zero out after + // persistence and add 1 to the count to avoid this operation guaranteeing we need another + // autoRotateCheckInterval later. + newEncs := upe + 1 + activeKey.Encryptions += uint64(newEncs) + newKeyring := b.keyring.Clone() + err := b.persistKeyring(ctx, newKeyring) + if err != nil { + return err + } + b.UnaccountedEncryptions.Sub(newEncs) + } + } + return nil +} + +// Mostly for testing, returns the total number of encryption operations performed on the active term +func (b *AESGCMBarrier) encryptions() int64 { + if b.keyring != nil { + activeKey := b.keyring.ActiveKey() + if activeKey != nil { + return b.UnaccountedEncryptions.Load() + int64(activeKey.Encryptions) + } + } + return 0 +} diff --git a/vault/barrier_aes_gcm_test.go b/vault/barrier_aes_gcm_test.go index b63f15b53d..035beb8688 100644 --- a/vault/barrier_aes_gcm_test.go +++ b/vault/barrier_aes_gcm_test.go @@ -6,6 +6,7 @@ import ( "crypto/rand" "encoding/json" "testing" + "time" log "github.com/hashicorp/go-hclog" "github.com/hashicorp/vault/sdk/helper/logging" @@ -594,5 +595,37 @@ func TestAESGCMBarrier_ReloadKeyring(t *testing.T) { if len(b.cache) != 0 { t.Fatal("failed to clear cache") } - +} + +func TestBarrier_LegacyRotate(t *testing.T) { + inm, err := inmem.NewInmem(nil, logger) + if err != nil { + t.Fatalf("err: %v", err) + } + b1, err := NewAESGCMBarrier(inm) + if err != nil { + t.Fatalf("err: %v", err) + } // Initialize the barrier + key, _ := b1.GenerateKey(rand.Reader) + b1.Initialize(context.Background(), key, nil, rand.Reader) + err = b1.Unseal(context.Background(), key) + if err != nil { + t.Fatalf("err: %v", err) + } + + k1 := b1.keyring.TermKey(1) + k1.Encryptions = 0 + k1.InstallTime = time.Now().Add(-24 * 366 * time.Hour) + b1.persistKeyring(context.Background(), b1.keyring) + b1.Seal() + + err = b1.Unseal(context.Background(), key) + if err != nil { + t.Fatalf("err: %v", err) + } + + reason, err := b1.CheckBarrierAutoRotate(context.Background()) + if err != nil || reason != legacyRotateReason { + t.Fail() + } } diff --git a/vault/barrier_test.go b/vault/barrier_test.go index 5468fce3ee..3385ec082d 100644 --- a/vault/barrier_test.go +++ b/vault/barrier_test.go @@ -11,115 +11,7 @@ import ( ) func testBarrier(t *testing.T, b SecurityBarrier) { - // Should not be initialized - init, err := b.Initialized(context.Background()) - if err != nil { - t.Fatalf("err: %v", err) - } - if init { - t.Fatalf("should not be initialized") - } - - // Should start sealed - sealed, err := b.Sealed() - if err != nil { - t.Fatalf("err: %v", err) - } - if !sealed { - t.Fatalf("should be sealed") - } - - // Sealing should be a no-op - if err := b.Seal(); err != nil { - t.Fatalf("err: %v", err) - } - - // All operations should fail - e := &logical.StorageEntry{Key: "test", Value: []byte("test")} - if err := b.Put(context.Background(), e); err != ErrBarrierSealed { - t.Fatalf("err: %v", err) - } - if _, err := b.Get(context.Background(), "test"); err != ErrBarrierSealed { - t.Fatalf("err: %v", err) - } - if err := b.Delete(context.Background(), "test"); err != ErrBarrierSealed { - t.Fatalf("err: %v", err) - } - if _, err := b.List(context.Background(), ""); err != ErrBarrierSealed { - t.Fatalf("err: %v", err) - } - - // Get a new key - key, err := b.GenerateKey(rand.Reader) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Validate minimum key length - min, max := b.KeyLength() - if min < 16 { - t.Fatalf("minimum key size too small: %d", min) - } - if max < min { - t.Fatalf("maximum key size smaller than min") - } - - // Unseal should not work - if err := b.Unseal(context.Background(), key); err != ErrBarrierNotInit { - t.Fatalf("err: %v", err) - } - - // Initialize the vault - if err := b.Initialize(context.Background(), key, nil, rand.Reader); err != nil { - t.Fatalf("err: %v", err) - } - - // Double Initialize should fail - if err := b.Initialize(context.Background(), key, nil, rand.Reader); err != ErrBarrierAlreadyInit { - t.Fatalf("err: %v", err) - } - - // Should be initialized - init, err = b.Initialized(context.Background()) - if err != nil { - t.Fatalf("err: %v", err) - } - if !init { - t.Fatalf("should be initialized") - } - - // Should still be sealed - sealed, err = b.Sealed() - if err != nil { - t.Fatalf("err: %v", err) - } - if !sealed { - t.Fatalf("should sealed") - } - - // Unseal should work - if err := b.Unseal(context.Background(), key); err != nil { - t.Fatalf("err: %v", err) - } - - // Unseal should no-op when done twice - if err := b.Unseal(context.Background(), key); err != nil { - t.Fatalf("err: %v", err) - } - - // Should no longer be sealed - sealed, err = b.Sealed() - if err != nil { - t.Fatalf("err: %v", err) - } - if sealed { - t.Fatalf("should be unsealed") - } - - // Verify the master key - if err := b.VerifyMaster(key); err != nil { - t.Fatalf("err: %v", err) - } + err, e, key := testInitAndUnseal(t, b) // Operations should work out, err := b.Get(context.Background(), "test") @@ -244,6 +136,119 @@ func testBarrier(t *testing.T, b SecurityBarrier) { } } +func testInitAndUnseal(t *testing.T, b SecurityBarrier) (error, *logical.StorageEntry, []byte) { + // Should not be initialized + init, err := b.Initialized(context.Background()) + if err != nil { + t.Fatalf("err: %v", err) + } + if init { + t.Fatalf("should not be initialized") + } + + // Should start sealed + sealed, err := b.Sealed() + if err != nil { + t.Fatalf("err: %v", err) + } + if !sealed { + t.Fatalf("should be sealed") + } + + // Sealing should be a no-op + if err := b.Seal(); err != nil { + t.Fatalf("err: %v", err) + } + + // All operations should fail + e := &logical.StorageEntry{Key: "test", Value: []byte("test")} + if err := b.Put(context.Background(), e); err != ErrBarrierSealed { + t.Fatalf("err: %v", err) + } + if _, err := b.Get(context.Background(), "test"); err != ErrBarrierSealed { + t.Fatalf("err: %v", err) + } + if err := b.Delete(context.Background(), "test"); err != ErrBarrierSealed { + t.Fatalf("err: %v", err) + } + if _, err := b.List(context.Background(), ""); err != ErrBarrierSealed { + t.Fatalf("err: %v", err) + } + + // Get a new key + key, err := b.GenerateKey(rand.Reader) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Validate minimum key length + min, max := b.KeyLength() + if min < 16 { + t.Fatalf("minimum key size too small: %d", min) + } + if max < min { + t.Fatalf("maximum key size smaller than min") + } + + // Unseal should not work + if err := b.Unseal(context.Background(), key); err != ErrBarrierNotInit { + t.Fatalf("err: %v", err) + } + + // Initialize the vault + if err := b.Initialize(context.Background(), key, nil, rand.Reader); err != nil { + t.Fatalf("err: %v", err) + } + + // Double Initialize should fail + if err := b.Initialize(context.Background(), key, nil, rand.Reader); err != ErrBarrierAlreadyInit { + t.Fatalf("err: %v", err) + } + + // Should be initialized + init, err = b.Initialized(context.Background()) + if err != nil { + t.Fatalf("err: %v", err) + } + if !init { + t.Fatalf("should be initialized") + } + + // Should still be sealed + sealed, err = b.Sealed() + if err != nil { + t.Fatalf("err: %v", err) + } + if !sealed { + t.Fatalf("should sealed") + } + + // Unseal should work + if err := b.Unseal(context.Background(), key); err != nil { + t.Fatalf("err: %v", err) + } + + // Unseal should no-op when done twice + if err := b.Unseal(context.Background(), key); err != nil { + t.Fatalf("err: %v", err) + } + + // Should no longer be sealed + sealed, err = b.Sealed() + if err != nil { + t.Fatalf("err: %v", err) + } + if sealed { + t.Fatalf("should be unsealed") + } + + // Verify the master key + if err := b.VerifyMaster(key); err != nil { + t.Fatalf("err: %v", err) + } + return err, e, key +} + func testBarrier_Rotate(t *testing.T, b SecurityBarrier) { // Initialize the barrier key, _ := b.GenerateKey(rand.Reader) diff --git a/vault/core.go b/vault/core.go index 45de53c4c1..1955176090 100644 --- a/vault/core.go +++ b/vault/core.go @@ -556,6 +556,8 @@ type Core struct { // for standby instances before we delete the upgrade keys keyRotateGracePeriod *int64 + autoRotateCancel context.CancelFunc + // number of workers to use for lease revocation in the expiration manager numExpirationWorkers int @@ -1761,6 +1763,11 @@ func (c *Core) sealInternalWithOptions(grabStateLock, keepHALock, performCleanup c.logger.Info("marked as sealed") + // Give the barrier a chance to persist encryption counts + if c.autoRotateCancel != nil { + c.checkBarrierAutoRotate(c.activeContext) + } + // Clear forwarding clients c.requestForwardingConnectionLock.Lock() c.clearForwardingClients() @@ -1904,6 +1911,13 @@ func (s standardUnsealStrategy) unseal(ctx context.Context, logger log.Logger, c if err := c.persistFeatureFlags(ctx); err != nil { return err } + + } + + if c.autoRotateCancel == nil { + var autoRotateCtx context.Context + autoRotateCtx, c.autoRotateCancel = context.WithCancel(c.activeContext) + go c.autoRotateBarrierLoop(autoRotateCtx) } if !c.IsDRSecondary() { @@ -2134,6 +2148,11 @@ func (c *Core) preSeal() error { result = multierror.Append(result, err) } + if c.autoRotateCancel != nil { + c.autoRotateCancel() + c.autoRotateCancel = nil + } + preSealPhysical(c) c.logger.Info("pre-seal teardown complete") @@ -2728,6 +2747,50 @@ func (c *Core) SetKeyRotateGracePeriod(t time.Duration) { atomic.StoreInt64(c.keyRotateGracePeriod, int64(t)) } +// Periodically test whether to automatically rotate the barrier key +func (c *Core) autoRotateBarrierLoop(ctx context.Context) { + t := time.NewTicker(autoRotateCheckInterval) + for { + select { + case <-t.C: + c.checkBarrierAutoRotate(ctx) + case <-ctx.Done(): + t.Stop() + return + } + } +} + +func (c *Core) checkBarrierAutoRotate(ctx context.Context) { + if c.isPrimary() { + reason, err := c.barrier.CheckBarrierAutoRotate(ctx) + if err != nil { + lf := c.logger.Error + if strings.HasSuffix(err.Error(), "context canceled") { + lf = c.logger.Debug + } + lf("error in barrier auto rotation", "error", err) + return + } + if reason != "" { + // Time to rotate. Invoke the rotation handler in order to both rotate and create + // the replication canary + c.logger.Info("automatic barrier key rotation triggered", "reason", reason) + + _, err := c.systemBackend.handleRotate(ctx, nil, nil) + if err != nil { + c.logger.Error("error automatically rotating barrier key", "error", err) + } else { + metrics.IncrCounter(barrierRotationsMetric, 1) + } + } + } +} + +func (c *Core) isPrimary() bool { + return !c.ReplicationState().HasState(consts.ReplicationPerformanceSecondary | consts.ReplicationDRSecondary) +} + func ParseRequiredState(raw string, hmacKey []byte) (*logical.WALState, error) { cooked, err := base64.StdEncoding.DecodeString(raw) if err != nil { diff --git a/vault/core_metrics.go b/vault/core_metrics.go index 7d3b2e7d2e..04007a2308 100644 --- a/vault/core_metrics.go +++ b/vault/core_metrics.go @@ -10,6 +10,7 @@ import ( "github.com/armon/go-metrics" "github.com/hashicorp/vault/helper/metricsutil" "github.com/hashicorp/vault/helper/namespace" + "github.com/hashicorp/vault/sdk/helper/consts" "github.com/hashicorp/vault/sdk/logical" ) @@ -69,8 +70,19 @@ func (c *Core) metricsLoop(stopCh chan struct{}) { continue } if c.perfStandby { // already have lock here, do not re-acquire - syncCounter(c) + err := syncCounters(c) + if err != nil { + c.logger.Error("writing syncing counters", "err", err) + } } else { + // Perf standbys will have synced above, but active nodes on a secondary cluster still need to ship + // barrier encryption counts + if c.ReplicationState().HasState(consts.ReplicationPerformanceSecondary) { + err := syncBarrierEncryptionCounter(c) + if err != nil { + c.logger.Error("writing syncing encryption counts", "err", err) + } + } err := c.saveCurrentRequestCounters(context.Background(), time.Now()) if err != nil { c.logger.Error("writing request counters to barrier", "err", err) diff --git a/vault/keyring.go b/vault/keyring.go index 8778ff2c2e..fb2ea0d8af 100644 --- a/vault/keyring.go +++ b/vault/keyring.go @@ -4,13 +4,29 @@ import ( "bytes" "encoding/json" "fmt" - "sync/atomic" "time" "github.com/hashicorp/errwrap" "github.com/hashicorp/vault/sdk/helper/jsonutil" ) +const ( + // 10% shy of the NIST recommended maximum, leaving a buffer to account for + // tracking losses. + absoluteOperationMaximum = int64(3_865_470_566) + absoluteOperationMinimum = int64(1_000_000) + minimumRotationInterval = 24 * time.Hour +) + +var ( + defaultRotationConfig = KeyRotationConfig{ + MaxOperations: absoluteOperationMaximum, + } + disabledRotationConfig = KeyRotationConfig{ + Disabled: true, + } +) + // Keyring is used to manage multiple encryption keys used by // the barrier. New keys can be installed and each has a sequential term. // The term used to encrypt a key is prefixed to the key written out. @@ -20,25 +36,32 @@ import ( // when a new key is added to the keyring, we can encrypt with the master key // and write out the new keyring. type Keyring struct { - masterKey []byte - keys map[uint32]*Key - activeTerm uint32 + masterKey []byte + keys map[uint32]*Key + activeTerm uint32 + rotationConfig KeyRotationConfig } // EncodedKeyring is used for serialization of the keyring type EncodedKeyring struct { - MasterKey []byte - Keys []*Key + MasterKey []byte + Keys []*Key + RotationConfig KeyRotationConfig } // Key represents a single term, along with the key used. type Key struct { - Term uint32 - Version int - Value []byte - InstallTime time.Time - Encryptions uint64 - ReportedEncryptions uint64 `json:",omitempty"` + Term uint32 + Version int + Value []byte + InstallTime time.Time + Encryptions uint64 `json:"encryptions,omitempty"` +} + +type KeyRotationConfig struct { + Disabled bool + MaxOperations int64 + Interval time.Duration } // Serialize is used to create a byte encoded key @@ -58,8 +81,9 @@ func DeserializeKey(buf []byte) (*Key, error) { // NewKeyring creates a new keyring func NewKeyring() *Keyring { k := &Keyring{ - keys: make(map[uint32]*Key), - activeTerm: 0, + keys: make(map[uint32]*Key), + activeTerm: 0, + rotationConfig: defaultRotationConfig, } return k } @@ -67,9 +91,10 @@ func NewKeyring() *Keyring { // Clone returns a new copy of the keyring func (k *Keyring) Clone() *Keyring { clone := &Keyring{ - masterKey: k.masterKey, - keys: make(map[uint32]*Key, len(k.keys)), - activeTerm: k.activeTerm, + masterKey: k.masterKey, + keys: make(map[uint32]*Key, len(k.keys)), + activeTerm: k.activeTerm, + rotationConfig: k.rotationConfig, } for idx, key := range k.keys { clone.keys[idx] = key @@ -102,6 +127,14 @@ func (k *Keyring) AddKey(key *Key) (*Keyring, error) { if key.Term > clone.activeTerm { clone.activeTerm = key.Term } + + // Zero out encryption estimates for previous terms + for term, key := range clone.keys { + if term != clone.activeTerm { + key.Encryptions = 0 + } + } + return clone, nil } @@ -156,7 +189,8 @@ func (k *Keyring) MasterKey() []byte { func (k *Keyring) Serialize() ([]byte, error) { // Create the encoded entry enc := EncodedKeyring{ - MasterKey: k.masterKey, + MasterKey: k.masterKey, + RotationConfig: k.rotationConfig, } for _, key := range k.keys { enc.Keys = append(enc.Keys, key) @@ -205,9 +239,15 @@ func (k *Keyring) Zeroize(keysToo bool) { } } -func (k *Keyring) AddEncryptionEstimate(term uint32, delta uint64) { - key := k.TermKey(term) - if key != nil { - atomic.AddUint64(&key.Encryptions, delta) +func (c *KeyRotationConfig) Sanitize() { + if c.MaxOperations == 0 || c.MaxOperations > absoluteOperationMaximum || c.MaxOperations < absoluteOperationMinimum { + c.MaxOperations = absoluteOperationMaximum + } + if c.Interval > 0 && c.Interval < minimumRotationInterval { + c.Interval = minimumRotationInterval } } + +func (c *KeyRotationConfig) Equals(config KeyRotationConfig) bool { + return c.MaxOperations == config.MaxOperations && c.Interval == config.Interval +} diff --git a/vault/logical_system.go b/vault/logical_system.go index 2f79779528..16b5fe4898 100644 --- a/vault/logical_system.go +++ b/vault/logical_system.go @@ -2525,52 +2525,82 @@ func (b *SystemBackend) handleKeyStatus(ctx context.Context, req *logical.Reques Data: map[string]interface{}{ "term": info.Term, "install_time": info.InstallTime.Format(time.RFC3339Nano), + "encryptions": info.Encryptions, }, } return resp, nil } +// handleKeyRotationConfigRead returns the barrier key rotation config +func (b *SystemBackend) handleKeyRotationConfigRead(_ context.Context, _ *logical.Request, _ *framework.FieldData) (*logical.Response, error) { + // Get the key info + rotConfig, err := b.Core.barrier.RotationConfig() + if err != nil { + return nil, err + } + + resp := &logical.Response{ + Data: map[string]interface{}{ + "max_operations": rotConfig.MaxOperations, + "enabled": !rotConfig.Disabled, + }, + } + if rotConfig.Interval > 0 { + resp.Data["interval"] = rotConfig.Interval.String() + } else { + resp.Data["interval"] = 0 + } + return resp, nil +} + +// handleKeyRotationConfigRead returns the barrier key rotation config +func (b *SystemBackend) handleKeyRotationConfigUpdate(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { + rotConfig, err := b.Core.barrier.RotationConfig() + if err != nil { + return nil, err + } + maxOps, ok, err := data.GetOkErr("max_operations") + if err != nil { + return nil, err + } + if ok { + rotConfig.MaxOperations = int64(maxOps.(int)) + } + interval, ok, err := data.GetOkErr("interval") + if err != nil { + return nil, err + } + if ok { + rotConfig.Interval = time.Second * time.Duration(interval.(int)) + } + + enabled, ok, err := data.GetOkErr("enabled") + if err != nil { + return nil, err + } + if ok { + rotConfig.Disabled = !enabled.(bool) + } + // Store the rotation config + b.Core.barrier.SetRotationConfig(ctx, rotConfig) + if err != nil { + return nil, err + } + + return nil, nil +} + // handleRotate is used to trigger a key rotation -func (b *SystemBackend) handleRotate(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { +func (b *SystemBackend) handleRotate(ctx context.Context, _ *logical.Request, _ *framework.FieldData) (*logical.Response, error) { repState := b.Core.ReplicationState() if repState.HasState(consts.ReplicationPerformanceSecondary) { return logical.ErrorResponse("cannot rotate on a replication secondary"), nil } - // Rotate to the new term - newTerm, err := b.Core.barrier.Rotate(ctx, b.Core.secureRandomReader) - if err != nil { - b.Backend.Logger().Error("failed to create new encryption key", "error", err) + if err := b.rotateBarrierKey(ctx); err != nil { + b.Backend.Logger().Error("error handling key rotation", "error", err) return handleError(err) } - b.Backend.Logger().Info("installed new encryption key") - - // In HA mode, we need to an upgrade path for the standby instances - if b.Core.ha != nil { - // Create the upgrade path to the new term - if err := b.Core.barrier.CreateUpgrade(ctx, newTerm); err != nil { - b.Backend.Logger().Error("failed to create new upgrade", "term", newTerm, "error", err) - } - - // Schedule the destroy of the upgrade path - time.AfterFunc(b.Core.KeyRotateGracePeriod(), func() { - b.Backend.Logger().Debug("cleaning up upgrade keys", "waited", b.Core.KeyRotateGracePeriod()) - if err := b.Core.barrier.DestroyUpgrade(b.Core.activeContext, newTerm); err != nil { - b.Backend.Logger().Error("failed to destroy upgrade", "term", newTerm, "error", err) - } - }) - } - - // Write to the canary path, which will force a synchronous truing during - // replication - if err := b.Core.barrier.Put(ctx, &logical.StorageEntry{ - Key: coreKeyringCanaryPath, - Value: []byte(fmt.Sprintf("new-rotation-term-%d", newTerm)), - }); err != nil { - b.Core.logger.Error("error saving keyring canary", "error", err) - return nil, errwrap.Wrapf("failed to save keyring canary: {{err}}", err) - } - return nil, nil } @@ -3842,6 +3872,44 @@ func (b *SystemBackend) verifyDROperationTokenOnSecondary(f framework.OperationF return f } +func (b *SystemBackend) rotateBarrierKey(ctx context.Context) error { + // Rotate to the new term + newTerm, err := b.Core.barrier.Rotate(ctx, b.Core.secureRandomReader) + if err != nil { + return errwrap.Wrap(errors.New("failed to create new encryption key"), err) + } + b.Backend.Logger().Info("installed new encryption key") + + // In HA mode, we need to an upgrade path for the standby instances + if b.Core.ha != nil { + // Create the upgrade path to the new term + if err := b.Core.barrier.CreateUpgrade(ctx, newTerm); err != nil { + b.Backend.Logger().Error("failed to create new upgrade", "term", newTerm, "error", err) + } + + // Schedule the destroy of the upgrade path + time.AfterFunc(b.Core.KeyRotateGracePeriod(), func() { + b.Backend.Logger().Debug("cleaning up upgrade keys", "waited", b.Core.KeyRotateGracePeriod()) + if err := b.Core.barrier.DestroyUpgrade(b.Core.activeContext, newTerm); err != nil { + b.Backend.Logger().Error("failed to destroy upgrade", "term", newTerm, "error", err) + } + }) + } + + // Write to the canary path, which will force a synchronous truing during + // replication + if err := b.Core.barrier.Put(ctx, &logical.StorageEntry{ + Key: coreKeyringCanaryPath, + Value: []byte(fmt.Sprintf("new-rotation-term-%d", newTerm)), + }); err != nil { + b.Core.logger.Error("error saving keyring canary", "error", err) + return errwrap.Wrap(errors.New("failed to save keyring canary"), err) + } + + return nil + +} + func sanitizePath(path string) string { if !strings.HasSuffix(path, "/") { path += "/" @@ -4339,6 +4407,25 @@ Enable a new audit backend or disable an existing backend. `, }, + "rotate-config": { + "Configures settings related to the backend encryption key management.", + ` + Configures settings related to the automatic rotation of the backend encryption key. + `, + }, + + "rotation-enabled": { + "Whether automatic rotation is enabled.", + "", + }, + "rotation-max-operations": { + "The number of encryption operations performed before the barrier key is automatically rotated.", + "", + }, + "rotation-interval": { + "How long after installation of an active key term that the key will be automatically rotated.", + "", + }, "rotate": { "Rotates the backend encryption key used to persist data.", ` diff --git a/vault/logical_system_paths.go b/vault/logical_system_paths.go index 384a891c6a..7fb6539922 100644 --- a/vault/logical_system_paths.go +++ b/vault/logical_system_paths.go @@ -602,6 +602,31 @@ func (b *SystemBackend) sealPaths() []*framework.Path { HelpDescription: strings.TrimSpace(sysHelp["key-status"][1]), }, + { + Pattern: "rotate/config$", + Fields: map[string]*framework.FieldSchema{ + "enabled": &framework.FieldSchema{ + Type: framework.TypeBool, + Description: strings.TrimSpace(sysHelp["rotation-enabled"][0]), + }, + "max_operations": &framework.FieldSchema{ + Type: framework.TypeInt, //64? + Description: strings.TrimSpace(sysHelp["rotation-max-operations"][0]), + }, + "interval": &framework.FieldSchema{ + Type: framework.TypeDurationSecond, + Description: strings.TrimSpace(sysHelp["rotation-interval"][0]), + }, + }, + Callbacks: map[logical.Operation]framework.OperationFunc{ + logical.ReadOperation: b.handleKeyRotationConfigRead, + logical.UpdateOperation: b.handleKeyRotationConfigUpdate, + }, + + HelpSynopsis: strings.TrimSpace(sysHelp["rotate-config"][0]), + HelpDescription: strings.TrimSpace(sysHelp["rotate-config"][1]), + }, + { Pattern: "rotate$", diff --git a/vault/logical_system_test.go b/vault/logical_system_test.go index 39feb4e4b1..7296d05bf0 100644 --- a/vault/logical_system_test.go +++ b/vault/logical_system_test.go @@ -2044,6 +2044,50 @@ func TestSystemBackend_keyStatus(t *testing.T) { "term": 1, } delete(resp.Data, "install_time") + delete(resp.Data, "encryptions") + if !reflect.DeepEqual(resp.Data, exp) { + t.Fatalf("got: %#v expect: %#v", resp.Data, exp) + } +} + +func TestSystemBackend_rotateConfig(t *testing.T) { + b := testSystemBackend(t) + req := logical.TestRequest(t, logical.ReadOperation, "rotate/config") + resp, err := b.HandleRequest(namespace.RootContext(nil), req) + if err != nil { + t.Fatalf("err: %v", err) + } + + exp := map[string]interface{}{ + "max_operations": absoluteOperationMaximum, + "interval": 0, + "enabled": true, + } + if !reflect.DeepEqual(resp.Data, exp) { + t.Fatalf("got: %#v expect: %#v", resp.Data, exp) + } + + req2 := logical.TestRequest(t, logical.UpdateOperation, "rotate/config") + req2.Data["max_operations"] = 2345678910 + req2.Data["interval"] = "5432h0m0s" + req2.Data["enabled"] = false + + resp, err = b.HandleRequest(namespace.RootContext(nil), req2) + if err != nil { + t.Fatalf("err: %v", err) + } + + resp, err = b.HandleRequest(namespace.RootContext(nil), req) + if err != nil { + t.Fatalf("err: %v", err) + } + + exp = map[string]interface{}{ + "max_operations": int64(2345678910), + "interval": "5432h0m0s", + "enabled": false, + } + if !reflect.DeepEqual(resp.Data, exp) { t.Fatalf("got: %#v expect: %#v", resp.Data, exp) } @@ -2071,6 +2115,7 @@ func TestSystemBackend_rotate(t *testing.T) { "term": 2, } delete(resp.Data, "install_time") + delete(resp.Data, "encryptions") if !reflect.DeepEqual(resp.Data, exp) { t.Fatalf("got: %#v expect: %#v", resp.Data, exp) } diff --git a/vault/request_handling_util.go b/vault/request_handling_util.go index ee01d243ee..a0634f5052 100644 --- a/vault/request_handling_util.go +++ b/vault/request_handling_util.go @@ -26,7 +26,12 @@ func shouldForward(c *Core, resp *logical.Response, err error) bool { return false } -func syncCounter(c *Core) { +func syncCounters(c *Core) error { + return nil +} + +func syncBarrierEncryptionCounter(c *Core) error { + return nil } func couldForward(c *Core) bool { diff --git a/vendor/github.com/hashicorp/vault/api/sys_rotate.go b/vendor/github.com/hashicorp/vault/api/sys_rotate.go index c525feb00d..e081587b11 100644 --- a/vendor/github.com/hashicorp/vault/api/sys_rotate.go +++ b/vendor/github.com/hashicorp/vault/api/sys_rotate.go @@ -68,10 +68,24 @@ func (c *Sys) KeyStatus() (*KeyStatus, error) { } result.InstallTime = installTime + encryptionsRaw, ok := secret.Data["encryptions"] + if ok { + encryptions, ok := encryptionsRaw.(json.Number) + if !ok { + return nil, errors.New("could not convert encryptions to a number") + } + encryptions64, err := encryptions.Int64() + if err != nil { + return nil, err + } + result.Encryptions = int(encryptions64) + } + return &result, err } type KeyStatus struct { Term int `json:"term"` InstallTime time.Time `json:"install_time"` + Encryptions int `json:"encryptions"` } diff --git a/vendor/github.com/hashicorp/vault/sdk/physical/file/file.go b/vendor/github.com/hashicorp/vault/sdk/physical/file/file.go index 736ec41597..d08d1c2b67 100644 --- a/vendor/github.com/hashicorp/vault/sdk/physical/file/file.go +++ b/vendor/github.com/hashicorp/vault/sdk/physical/file/file.go @@ -225,9 +225,14 @@ func (b *FileBackend) PutInternal(ctx context.Context, entry *physical.Entry) er if err := b.validatePath(entry.Key); err != nil { return err } - path, key := b.expandPath(entry.Key) + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + // Make the parent tree if err := os.MkdirAll(path, 0700); err != nil { return err @@ -249,12 +254,6 @@ func (b *FileBackend) PutInternal(ctx context.Context, entry *physical.Entry) er return errors.New("could not successfully get a file handle") } - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - enc := json.NewEncoder(f) encErr := enc.Encode(&fileEntry{ Value: entry.Value,