Diagnose Storage Panic Bugfixes (#11923)

* partial

* fix raft panics and ensure checks are skipped if storage isnt initialized

* cleanup directories

* newline

* typo in nil check

* another nil check

command/operator_diagnose.go

@@ -269,6 +269,12 @@ func (c *OperatorDiagnoseCommand) offlineDiagnostics(ctx context.Context) error
 			return nil
 		})
 
+		if backend == nil {
+			diagnose.Fail(ctx, "Diagnose could not initialize storage backend.")
+			span.End()
+			return fmt.Errorf("Diagnose could not initialize storage backend.")
+		}
+
 		// Check for raft quorum status
 		if config.Storage.Type == storageTypeRaft {
 			path := os.Getenv(raft.EnvVaultRaftPath)
@@ -346,6 +352,11 @@ func (c *OperatorDiagnoseCommand) offlineDiagnostics(ctx context.Context) error
 		return nil
 	})
 
+	// Return from top-level span when backend is nil
+	if backend == nil {
+		return fmt.Errorf("Diagnose could not initialize storage backend.")
+	}
+
 	var configSR sr.ServiceRegistration
 	diagnose.Test(ctx, "service-discovery", func(ctx context.Context) error {
 		if config.ServiceRegistration == nil || config.ServiceRegistration.Config == nil {
@@ -423,10 +434,6 @@ SEALFAIL:
 			return diagnose.SpotError(ctx, "init-randreader", err)
 		}
 		diagnose.SpotOk(ctx, "init-randreader", "")
-
-		if backend == nil {
-			return fmt.Errorf(BackendUninitializedErr)
-		}
 		coreConfig = createCoreConfig(server, config, *backend, configSR, barrierSeal, unwrapSeal, metricsHelper, metricSink, secureRandomReader)
 		return nil
 	}); err != nil {
@@ -435,9 +442,6 @@ SEALFAIL:
 
 	var disableClustering bool
 	diagnose.Test(ctx, "setup-ha-storage", func(ctx context.Context) error {
-		if backend == nil {
-			return fmt.Errorf(BackendUninitializedErr)
-		}
 		diagnose.Test(ctx, "create-ha-storage-backend", func(ctx context.Context) error {
 			// Initialize the separate HA storage backend, if it exists
 			disableClustering, err = initHaBackend(server, config, &coreConfig, *backend)

command/operator_diagnose_test.go

@@ -6,6 +6,7 @@ import (
 	"context"
 	"fmt"
 	"io/ioutil"
+	"os"
 	"strings"
 	"testing"
 
@@ -176,12 +177,11 @@ func TestOperatorDiagnoseCommand_Run(t *testing.T) {
 			[]*diagnose.Result{
 				{
 					Name:   "storage",
-					Status: diagnose.ErrorStatus,
+					Status: diagnose.WarningStatus,
 					Children: []*diagnose.Result{
 						{
-							Name:    "create-storage-backend",
-							Status:  diagnose.ErrorStatus,
-							Message: "failed to open bolt file",
+							Name:   "create-storage-backend",
+							Status: diagnose.OkStatus,
 						},
 						{
 							Name:    "raft folder permission checks",
@@ -190,8 +190,8 @@ func TestOperatorDiagnoseCommand_Run(t *testing.T) {
 						},
 						{
 							Name:    "raft quorum",
-							Status:  diagnose.ErrorStatus,
-							Message: "could not determine quorum status",
+							Status:  diagnose.WarningStatus,
+							Message: "even number of voters found",
 						},
 					},
 				},
@@ -378,6 +378,26 @@ func TestOperatorDiagnoseCommand_Run(t *testing.T) {
 				},
 			},
 		},
+		{
+			"diagnose_raft_no_folder_backend",
+			[]string{
+				"-config", "./server/test-fixtures/diagnose_raft_no_bolt_folder.hcl",
+			},
+			[]*diagnose.Result{
+				{
+					Name:    "storage",
+					Status:  diagnose.ErrorStatus,
+					Message: "Diagnose could not initialize storage backend.",
+					Children: []*diagnose.Result{
+						{
+							Name:    "create-storage-backend",
+							Status:  diagnose.ErrorStatus,
+							Message: "no such file or directory",
+						},
+					},
+				},
+			},
+		},
 	}
 
 	t.Run("validations", func(t *testing.T) {
@@ -459,5 +479,9 @@ func compareResult(exp *diagnose.Result, act *diagnose.Result) error {
 		return compareResults(exp.Children, act.Children)
 	}
 
+	// Remove raft file if it exists
+	os.Remove("./server/test-fixtures/vault.db")
+	os.RemoveAll("./server/test-fixtures/raft")
+
 	return nil
 }

command/server/test-fixtures/config_diagnose_ok.hcl

@@ -9,9 +9,7 @@ listener "tcp" {
 }
 
 backend "consul" {
-    address = "127.0.0.1:8500"
-    foo = "bar"
-    advertise_addr = "foo"
+    address = "127.0.0.1:1025"
 }
 
 ha_backend "consul" {

command/server/test-fixtures/config_raft.hcl

@@ -9,7 +9,7 @@ listener "tcp" {
 }
 
 storage "raft" {
-  path = "./server/test-fixtures/raft_storage_file.db"
+  path = "./server/test-fixtures/"
   node_id = "raft_node_1"
 }
 cluster_addr = "http://127.0.0.1:8201"

command/server/test-fixtures/diagnose_raft_no_bolt_folder.hcl

@@ -0,0 +1,8 @@
+storage "raft" {
+  path = "/path/to/raft/data"
+  node_id = "raft_node_1"
+}
+
+api_addr = "http://127.0.0.1:8200"
+cluster_addr = "https://127.0.0.1:8201"
+ui = true

physical/raft/raft.go

@@ -1014,6 +1014,11 @@ func (b *RaftBackend) GetConfigurationOffline() (*RaftConfigurationResponse, err
 	config := &RaftConfigurationResponse{
 		Index: state.Index,
 	}
+
+	if configuration == nil || configuration.Servers == nil {
+		return config, nil
+	}
+
 	for _, server := range configuration.Servers {
 		entry := &RaftServer{
 			NodeID:  server.Id,