core metrics changes (#30466)

* core metrics changes

* removing internal repo references

* CE changes from Vault Operator Import

* removing unrelated code

* removing unnecessary go mod updates

* updating go mod

vault/core_metrics.go

@@ -24,6 +24,10 @@ import (
 	uicustommessages "github.com/hashicorp/vault/vault/ui_custom_messages"
 )
 
+const (
+	KVv2MetadataPath = "metadata"
+)
+
 func (c *Core) metricsLoop(stopCh chan struct{}) {
 	emitTimer := time.Tick(time.Second)
 
@@ -453,19 +457,19 @@ func (c *Core) kvCollectionErrorCount() {
 	)
 }
 
-func (c *Core) walkKvMountSecrets(ctx context.Context, m *kvMount) {
-	var subdirectories []string
-	if m.Version == "1" {
-		subdirectories = []string{m.Namespace.Path + m.MountPoint}
-	} else {
-		subdirectories = []string{m.Namespace.Path + m.MountPoint + "metadata/"}
-	}
+func (c *Core) walkKvSecrets(
+	ctx context.Context,
+	rootDirs []string,
+	m *kvMount,
+	onSecret func(ctx context.Context, fullPath string) error,
+) error {
+	subdirectories := rootDirs
 
 	for len(subdirectories) > 0 {
-		// Check for cancellation
+		// Context cancellation check
 		select {
 		case <-ctx.Done():
-			return
+			return nil
 		default:
 			break
 		}
@@ -477,6 +481,7 @@ func (c *Core) walkKvMountSecrets(ctx context.Context, m *kvMount) {
 			Operation: logical.ListOperation,
 			Path:      currentDirectory,
 		}
+
 		resp, err := c.router.Route(ctx, listRequest)
 		if err != nil {
 			c.kvCollectionErrorCount()
@@ -491,11 +496,12 @@ func (c *Core) walkKvMountSecrets(ctx context.Context, m *kvMount) {
 				break
 			}
 			// Quit handling this mount point (but it'll still appear in the list)
-			return
+			return err
 		}
 		if resp == nil {
 			continue
 		}
+
 		rawKeys, ok := resp.Data["keys"]
 		if !ok {
 			continue
@@ -505,16 +511,22 @@ func (c *Core) walkKvMountSecrets(ctx context.Context, m *kvMount) {
 			c.kvCollectionErrorCount()
 			c.logger.Error("KV list keys are not a []string", "mount_point", m.MountPoint, "rawKeys", rawKeys)
 			// Quit handling this mount point (but it'll still appear in the list)
-			return
+			return fmt.Errorf("KV list keys are not a []string")
 		}
+
 		for _, path := range keys {
-			if len(path) > 0 && path[len(path)-1] == '/' {
-				subdirectories = append(subdirectories, currentDirectory+path)
+			fullPath := currentDirectory + path
+			if strings.HasSuffix(path, "/") {
+				subdirectories = append(subdirectories, fullPath)
 			} else {
-				m.NumSecrets += 1
+				if callBackErr := onSecret(ctx, fullPath); callBackErr != nil {
+					c.logger.Error("failed to get metadata for KVv2 secret", "path", fullPath, "error", err)
+					return callBackErr
+				}
 			}
 		}
 	}
+	return nil
 }
 
 // GetLocalAndReplicatedSecretMounts returns the number of replicated and local secret mounts
@@ -748,7 +760,7 @@ func (c *Core) GetSecretEngineUsageMetrics() map[string]int {
 	return mounts
 }
 
-// GetAuthMethodUsageMetrics returns a map of auth mount types to the number of those mounts that exist.
+// Get returns a map of auth mount types to the number of those mounts that exist.
 func (c *Core) GetAuthMethodUsageMetrics() map[string]int {
 	mounts := make(map[string]int)
 
@@ -839,6 +851,31 @@ func (c *Core) GetKvUsageMetrics(ctx context.Context, kvVersion string) (map[str
 	return results, nil
 }
 
+func (c *Core) walkKvMountSecrets(ctx context.Context, m *kvMount) {
+	var startDirs []string
+	if m.Version == "1" {
+		startDirs = []string{m.Namespace.Path + m.MountPoint}
+	} else {
+		startDirs = []string{m.Namespace.Path + m.MountPoint + KVv2MetadataPath + "/"}
+	}
+
+	err := c.walkKvSecrets(ctx, startDirs, m, func(ctx context.Context, fullPath string) error {
+		m.NumSecrets++
+		return nil
+	})
+	if err != nil {
+		// ErrUnsupportedPath probably means that the mount is not there anymore,
+		// don't log those cases.
+		if !strings.Contains(err.Error(), logical.ErrUnsupportedPath.Error()) &&
+			// ErrSetupReadOnly means the mount's currently being set up.
+			// Nothing is wrong and there's no cause for alarm, just that we can't get data from it
+			// yet. We also shouldn't log these cases
+			!strings.Contains(err.Error(), logical.ErrSetupReadOnly.Error()) {
+			c.logger.Error("failed to walk KV mount", "mount_point", m.MountPoint, "error", err)
+		}
+	}
+}
+
 func (c *Core) kvSecretGaugeCollector(ctx context.Context) ([]metricsutil.GaugeLabelValues, error) {
 	// Find all KV mounts
 	mounts := c.findKvMounts()