Backport Allow nodes to join a cluster with a multi-seal configuration into ce/main (#14426)

* Allow nodes to join a cluster with a multi-seal configuration (#14271)

* Move SealGenerationInfo validation logic to its own file.

Refactor methog SealGenerationInfo.Validate into function
ValidateSealGeneration.

* Refactor SealGeneationInfo.Validate to func ValidateMultiSealGenerationInfo.

* Allow nodes to join a cluster with a multi-seal configuration.

Relax the multi-seal restriction when setting the Vault seal: allow an initial
multi-seal configuration if there is no stored seal generation information.

Validate multi-seal configuration at initialization time, but do not allow for
an initial multi-seal configuration at this time.

* Add unit tests.

* Run make fmt. Add copyright header.

* Add changelog entry.

* Add godoc comments to unit tests.

* Add seal generation validation stub files.

---------

Co-authored-by: Victor Rodriguez Rizo <vrizo@hashicorp.com>

changelog/_14271.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+core/seal (enterprise): Make it possible for new nodes to join a cluster configured with Seal High Availability.
+```

command/server.go

@@ -560,7 +560,7 @@ func (c *ServerCommand) runRecoveryMode() int {
 		return 1
 	}
 
-	hasPartialPaths, err := hasPartiallyWrappedPaths(ctx, backend)
+	hasPartialPaths, err := vault.HasPartiallyWrappedPaths(ctx, backend)
 	if err != nil {
 		c.UI.Error(fmt.Sprintf("Cannot determine if there are partially seal wrapped entries in storage: %v", err))
 		return 1
@@ -1935,7 +1935,7 @@ func (c *ServerCommand) configureSeals(ctx context.Context, config *server.Confi
 		return nil, nil, fmt.Errorf("Error getting seal generation info: %v", err)
 	}
 
-	hasPartialPaths, err := hasPartiallyWrappedPaths(ctx, backend)
+	hasPartialPaths, err := vault.HasPartiallyWrappedPaths(ctx, backend)
 	if err != nil {
 		return nil, nil, fmt.Errorf("Cannot determine if there are partially seal wrapped entries in storage: %v", err)
 	}
@@ -2752,25 +2752,16 @@ func (c *ServerCommand) computeSealGenerationInfo(existingSealGenInfo *vaultseal
 		Enabled:    multisealEnabled,
 	}
 
-	if multisealEnabled || (existingSealGenInfo != nil && existingSealGenInfo.Enabled) {
-		err := newSealGenInfo.Validate(existingSealGenInfo, hasPartiallyWrappedPaths)
-		if err != nil {
-			return nil, err
-		}
+	// Validate multi seal concerns of the seal configuration. Note that at this
+	// point Vault is starting up, not initializing (as in "vault operator init").
+	err := vaultseal.ValidateMultiSealGenerationInfo(false, newSealGenInfo, existingSealGenInfo, hasPartiallyWrappedPaths)
+	if err != nil {
+		return nil, err
 	}
 
 	return newSealGenInfo, nil
 }
 
-func hasPartiallyWrappedPaths(ctx context.Context, backend physical.Backend) (bool, error) {
-	paths, err := vault.GetPartiallySealWrappedPaths(ctx, backend)
-	if err != nil {
-		return false, err
-	}
-
-	return len(paths) > 0, nil
-}
-
 func initHaBackend(c *ServerCommand, config *server.Config, coreConfig *vault.CoreConfig, backend physical.Backend) (bool, error) {
 	// Initialize the separate HA storage backend, if it exists
 	var ok bool

vault/core.go

@@ -1876,6 +1876,10 @@ func (c *Core) unsealFragment(key []byte, migrate bool) error {
 		return nil
 	}
 
+	if err := c.ValidateMultiSealConfig(ctx, false); err != nil {
+		return err
+	}
+
 	sealToUse := c.seal
 	if migrate {
 		c.logger.Info("unsealing using migration seal")

vault/init.go

@@ -163,6 +163,10 @@ func (c *Core) Initialize(ctx context.Context, initParams *InitParams) (*InitRes
 		return nil, err
 	}
 
+	if err := c.ValidateMultiSealConfig(ctx, true); err != nil {
+		return nil, err
+	}
+
 	atomic.StoreUint32(&initInProgress, 1)
 	defer atomic.StoreUint32(&initInProgress, 0)
 	barrierConfig := initParams.BarrierConfig
@@ -443,6 +447,23 @@ func (c *Core) Initialize(ctx context.Context, initParams *InitParams) (*InitRes
 	return results, nil
 }
 
+// ValidateMultiSealConfig is an utility method for verifying SealGenerationInfo.
+// Its purpose is to read the existing SealGenerationInfo from storage, if any,
+// and to determine whether there are partially wrapped paths.
+// Argument onInit indicates whether Vault is being initialized and thus creating
+// the initial barrier seal.
+func (c *Core) ValidateMultiSealConfig(ctx context.Context, onInit bool) error {
+	existingSgi, err := PhysicalSealGenInfo(ctx, c.PhysicalAccess())
+	if err != nil {
+		return fmt.Errorf("error reading existing seal generation info from storage: %w", err)
+	}
+	hasPartiallyWrappedPaths, err := HasPartiallyWrappedPaths(ctx, c.PhysicalAccess())
+	if err != nil {
+		return fmt.Errorf("cannot determine whether partially wrapped entries in storage: %w", err)
+	}
+	return seal.ValidateMultiSealGenerationInfo(onInit, c.seal.GetAccess().GetSealGenerationInfo(), existingSgi, hasPartiallyWrappedPaths)
+}
+
 // UnsealWithStoredKeys performs auto-unseal using stored keys. An error
 // return value of "nil" implies the Vault instance is unsealed.
 //

vault/raft.go

@@ -1022,6 +1022,8 @@ func (c *Core) getRaftChallenge(leaderInfo *raft.LeaderJoinInfo) (*raftInformati
 		return nil, err
 	}
 
+	// We compare here the local seal configuration to that of the leader we are trying to join,
+	// thus there is no need to call ValidateSealGenerationInfo.
 	if !CompatibleSealTypes(sealConfig.Type, c.seal.BarrierSealConfigType().String()) {
 		return nil, fmt.Errorf("incompatible seal types between raft leader (%s) and follower (%s)", sealConfig.Type, c.seal.BarrierSealConfigType())
 	}

vault/seal/seal.go

@@ -8,16 +8,13 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"os"
 	"reflect"
 	"sort"
-	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
 
 	metrics "github.com/armon/go-metrics"
-	"github.com/google/go-cmp/cmp"
 	"github.com/hashicorp/go-hclog"
 	wrapping "github.com/hashicorp/go-kms-wrapping/v2"
 	"github.com/hashicorp/go-kms-wrapping/v2/aead"
@@ -62,174 +59,6 @@ type SealGenerationInfo struct {
 	Enabled    bool
 }
 
-// Validate is used to sanity check the seal generation info being created
-func (sgi *SealGenerationInfo) Validate(existingSgi *SealGenerationInfo, hasPartiallyWrappedPaths bool) error {
-	existingSealsLen := 0
-	numConfiguredSeals := len(sgi.Seals)
-	configuredSealNameAndType := sealNameAndTypeAsStr(sgi.Seals)
-
-	// If no previous generation info exists, make sure we perform the initial migration/setup
-	// check for enabled configured seals to allow an old style seal migration configuration
-	if existingSgi == nil {
-		if numConfiguredSeals > 1 {
-			return fmt.Errorf("Initializing a cluster or enabling multi-seal on an existing "+
-				"cluster must occur with a single seal before adding additional seals\n"+
-				"Configured seals: %v", configuredSealNameAndType)
-		}
-
-		// No point in comparing anything more as we don't have any information around the
-		// existing seal if any actually existed
-		return nil
-	}
-
-	// Validate that we're in a safe spot with respect to disabling multiseal
-	if existingSgi.Enabled && !sgi.Enabled {
-		if len(existingSgi.Seals) > 1 {
-			return fmt.Errorf("multi-seal is disabled but previous configuration had multiple seals.  re-enable and migrate to a single seal before disabling multi-seal")
-		} else if !existingSgi.IsRewrapped() {
-			return fmt.Errorf("multi-seal is disabled but previous storage was not fully re-wrapped, re-enable multi-seal and allow rewrapping to complete before disabling multi-seal")
-		}
-	}
-
-	existingSealNameAndType := sealNameAndTypeAsStr(existingSgi.Seals)
-	previousShamirConfigured := false
-
-	if sgi.Generation == existingSgi.Generation {
-		if !haveMatchingSeals(sgi.Seals, existingSgi.Seals) {
-			return fmt.Errorf("existing seal generation is the same, but the configured seals are different\n"+
-				"Existing seals: %v\n"+
-				"Configured seals: %v", existingSealNameAndType, configuredSealNameAndType)
-		}
-		return nil
-	}
-
-	existingSealsLen = len(existingSgi.Seals)
-	for _, sealKmsConfig := range existingSgi.Seals {
-		if sealKmsConfig.Type == wrapping.WrapperTypeShamir.String() {
-			previousShamirConfigured = true
-			break
-		}
-	}
-
-	if !previousShamirConfigured && (!existingSgi.IsRewrapped() || hasPartiallyWrappedPaths) && os.Getenv("VAULT_SEAL_REWRAP_SAFETY") != "disable" {
-		return errors.New("cannot make seal config changes while seal re-wrap is in progress, please revert any seal configuration changes")
-	}
-
-	numSealsToAdd := 0
-	// With a previously configured shamir seal, we are either going from [shamir]->[auto]
-	// or [shamir]->[another shamir] (since we do not allow multiple shamir
-	// seals, and, mixed shamir and auto seals). Also, we do not allow shamir seals to
-	// be set disabled, so, the number of seals to add is always going to be the length
-	// of new seal configs.
-	if previousShamirConfigured {
-		numSealsToAdd = numConfiguredSeals
-	} else {
-		numSealsToAdd = numConfiguredSeals - existingSealsLen
-	}
-
-	numSealsToDelete := existingSealsLen - numConfiguredSeals
-	switch {
-	case numSealsToAdd > 1:
-		return fmt.Errorf("cannot add more than one seal\n"+
-			"Existing seals: %v\n"+
-			"Configured seals: %v", existingSealNameAndType, configuredSealNameAndType)
-
-	case numSealsToDelete > 1:
-		return fmt.Errorf("cannot delete more than one seal\n"+
-			"Existing seals: %v\n"+
-			"Configured seals: %v", existingSealNameAndType, configuredSealNameAndType)
-
-	case !previousShamirConfigured && existingSgi != nil && !haveCommonSeal(existingSgi.Seals, sgi.Seals):
-		// With a previously configured shamir seal, we are either going from [shamir]->[auto] or [shamir]->[another shamir],
-		// in which case we cannot have a common seal because shamir seals cannot be set to disabled, they can only be deleted.
-		return fmt.Errorf("must have at least one seal in common with the old generation\n"+
-			"Existing seals: %v\n"+
-			"Configured seals: %v", existingSealNameAndType, configuredSealNameAndType)
-	}
-	return nil
-}
-
-func sealNameAndTypeAsStr(seals []*configutil.KMS) string {
-	info := []string{}
-	for _, seal := range seals {
-		info = append(info, fmt.Sprintf("Name: %s Type: %s", seal.Name, seal.Type))
-	}
-	return fmt.Sprintf("[%s]", strings.Join(info, ", "))
-}
-
-// haveMatchingSeals verifies that we have the corresponding matching seals by name and type, config and other
-// properties are ignored in the comparison
-func haveMatchingSeals(existingSealKmsConfigs, newSealKmsConfigs []*configutil.KMS) bool {
-	if len(existingSealKmsConfigs) != len(newSealKmsConfigs) {
-		return false
-	}
-
-	for _, existingSealKmsConfig := range existingSealKmsConfigs {
-		found := false
-		for _, newSealKmsConfig := range newSealKmsConfigs {
-			if cmp.Equal(existingSealKmsConfig, newSealKmsConfig, compareKMSConfigByNameAndType()) {
-				found = true
-				break
-			}
-		}
-
-		if !found {
-			return false
-		}
-	}
-	return true
-}
-
-// haveCommonSeal verifies that we have at least one matching seal across
-// the inputs by name and type, config and other properties are ignored in
-// the comparison
-func haveCommonSeal(existingSealKmsConfigs, newSealKmsConfigs []*configutil.KMS) bool {
-	for _, existingSealKmsConfig := range existingSealKmsConfigs {
-		for _, newSealKmsConfig := range newSealKmsConfigs {
-			// Technically we might be matching the "wrong" seal if the old seal was renamed to
-			// "transit-disabled" and we have a new seal named transit. There isn't any way for
-			// us to properly distinguish between them
-			if cmp.Equal(existingSealKmsConfig, newSealKmsConfig, compareKMSConfigByNameAndType()) {
-				return true
-			}
-		}
-	}
-
-	// We might have renamed a disabled seal that was previously used so attempt to match by
-	// removing the "-disabled" suffix
-	for _, seal := range findRenamedDisabledSeals(newSealKmsConfigs) {
-		clonedSeal := seal.Clone()
-		clonedSeal.Name = strings.TrimSuffix(clonedSeal.Name, configutil.KmsRenameDisabledSuffix)
-
-		for _, existingSealKmsConfig := range existingSealKmsConfigs {
-			if cmp.Equal(existingSealKmsConfig, clonedSeal, compareKMSConfigByNameAndType()) {
-				return true
-			}
-		}
-	}
-
-	return false
-}
-
-func findRenamedDisabledSeals(configs []*configutil.KMS) []*configutil.KMS {
-	disabledSeals := []*configutil.KMS{}
-	for _, seal := range configs {
-		if seal.Disabled && strings.HasSuffix(seal.Name, configutil.KmsRenameDisabledSuffix) {
-			disabledSeals = append(disabledSeals, seal)
-		}
-	}
-	return disabledSeals
-}
-
-func compareKMSConfigByNameAndType() cmp.Option {
-	// We only match based on name and type to avoid configuration changes such
-	// as a Vault token change in the config map from eliminating the match and
-	// preventing startup on a matching seal.
-	return cmp.Comparer(func(a, b *configutil.KMS) bool {
-		return a.Name == b.Name && a.Type == b.Type
-	})
-}
-
 // SetRewrapped updates the SealGenerationInfo's rewrapped status to the provided value.
 func (sgi *SealGenerationInfo) SetRewrapped(value bool) {
 	sgi.rewrapped.Store(value)

vault/seal/seal_generation_validation_ce.go

@@ -0,0 +1,10 @@
+// Copyright IBM Corp. 2016, 2025
+// SPDX-License-Identifier: MPL-2.0
+
+//go:build !enterprise
+
+package seal
+
+func ValidateMultiSealGenerationInfo(_ bool, _, _ *SealGenerationInfo, _ bool) error {
+	return nil
+}

vault/seal_util_ce.go

@@ -0,0 +1,16 @@
+// Copyright IBM Corp. 2016, 2025
+// SPDX-License-Identifier: BUSL-1.1
+
+//go:build !enterprise
+
+package vault
+
+import (
+	"context"
+
+	"github.com/hashicorp/vault/sdk/physical"
+)
+
+func HasPartiallyWrappedPaths(_ context.Context, _ physical.Backend) (bool, error) {
+	return false, nil
+}