Merge pull request #840 from hashicorp/issue-395

Allow separate HA physical backend.

command/server.go

@@ -39,12 +39,13 @@ type ServerCommand struct {
 }
 
 func (c *ServerCommand) Run(args []string) int {
-	var dev bool
+	var dev, verifyOnly bool
 	var configPath []string
 	var logLevel string
 	flags := c.Meta.FlagSet("server", FlagSetDefault)
 	flags.BoolVar(&dev, "dev", false, "")
 	flags.StringVar(&logLevel, "log-level", "info", "")
+	flags.BoolVar(&verifyOnly, "verify-only", false, "")
 	flags.Usage = func() { c.Ui.Error(c.Help()) }
 	flags.Var((*sliceflag.StringFlag)(&configPath), "config", "config")
 	if err := flags.Parse(args); err != nil {
@@ -113,22 +114,9 @@ func (c *ServerCommand) Run(args []string) int {
 		return 1
 	}
 
-	// Attempt to detect the advertise address possible
+	coreConfig := &vault.CoreConfig{
-	if detect, ok := backend.(physical.AdvertiseDetect); ok && config.Backend.AdvertiseAddr == "" {
-		advertise, err := c.detectAdvertise(detect, config)
-		if err != nil {
-			c.Ui.Error(fmt.Sprintf("Error detecting advertise address: %s", err))
-		} else if advertise == "" {
-			c.Ui.Error("Failed to detect advertise address.")
-		} else {
-			config.Backend.AdvertiseAddr = advertise
-		}
-	}
-
-	// Initialize the core
-	core, err := vault.NewCore(&vault.CoreConfig{
-		AdvertiseAddr:      config.Backend.AdvertiseAddr,
 		Physical:           backend,
+		HAPhysical:         nil,
 		AuditBackends:      c.AuditBackends,
 		CredentialBackends: c.CredentialBackends,
 		LogicalBackends:    c.LogicalBackends,
@@ -137,7 +125,41 @@ func (c *ServerCommand) Run(args []string) int {
 		DisableMlock:       config.DisableMlock,
 		MaxLeaseTTL:        config.MaxLeaseTTL,
 		DefaultLeaseTTL:    config.DefaultLeaseTTL,
-	})
+	}
+
+	// Initialize the separate HA physical backend, if it exists
+	if config.HABackend != nil {
+		habackend, err := physical.NewBackend(
+			config.HABackend.Type, config.HABackend.Config)
+		if err != nil {
+			c.Ui.Error(fmt.Sprintf(
+				"Error initializing backend of type %s: %s",
+				config.HABackend.Type, err))
+			return 1
+		}
+
+		var ok bool
+		if coreConfig.HAPhysical, ok = habackend.(physical.HABackend); !ok {
+			c.Ui.Error("Specified HA backend does not support HA")
+			return 1
+		}
+		coreConfig.AdvertiseAddr = config.HABackend.AdvertiseAddr
+	}
+
+	// Attempt to detect the advertise address possible
+	if detect, ok := coreConfig.HAPhysical.(physical.AdvertiseDetect); ok && coreConfig.AdvertiseAddr == "" {
+		advertise, err := c.detectAdvertise(detect, config)
+		if err != nil {
+			c.Ui.Error(fmt.Sprintf("Error detecting advertise address: %s", err))
+		} else if advertise == "" {
+			c.Ui.Error("Failed to detect advertise address.")
+		} else {
+			coreConfig.AdvertiseAddr = advertise
+		}
+	}
+
+	// Initialize the core
+	core, err := vault.NewCore(coreConfig)
 	if err != nil {
 		c.Ui.Error(fmt.Sprintf("Error initializing core: %s", err))
 		return 1
@@ -186,11 +208,17 @@ func (c *ServerCommand) Run(args []string) int {
 		mlock.Supported(), !config.DisableMlock)
 	infoKeys = append(infoKeys, "log level", "mlock", "backend")
 
-	// If the backend supports HA, then note it
+	if config.HABackend != nil {
-	if _, ok := backend.(physical.HABackend); ok {
+		info["HA backend"] = config.HABackend.Type
-		info["backend"] += " (HA available)"
+		info["advertise address"] = coreConfig.AdvertiseAddr
-		info["advertise address"] = config.Backend.AdvertiseAddr
+		infoKeys = append(infoKeys, "HA backend", "advertise address")
-		infoKeys = append(infoKeys, "advertise address")
+	} else {
+		// If the backend supports HA, then note it
+		if coreConfig.HAPhysical != nil {
+			info["backend"] += " (HA available)"
+			info["advertise address"] = coreConfig.AdvertiseAddr
+			infoKeys = append(infoKeys, "advertise address")
+		}
 	}
 
 	// Initialize the telemetry
@@ -225,6 +253,10 @@ func (c *ServerCommand) Run(args []string) int {
 		lns = append(lns, ln)
 	}
 
+	if verifyOnly {
+		return 0
+	}
+
 	// Initialize the HTTP server
 	server := &http.Server{}
 	server.Handler = vaulthttp.Handler(core)

command/server/config.go

@@ -17,6 +17,7 @@ import (
 type Config struct {
 	Listeners []*Listener `hcl:"-"`
 	Backend   *Backend    `hcl:"-"`
+	HABackend *Backend    `hcl:"-"`
 
 	DisableCache bool `hcl:"disable_cache"`
 	DisableMlock bool `hcl:"disable_mlock"`
@@ -191,6 +192,12 @@ func LoadConfigFile(path string) (*Config, error) {
 			return nil, err
 		}
 	}
+	if objs := obj.Get("ha_backend", false); objs != nil {
+		result.HABackend, err = loadBackend(objs)
+		if err != nil {
+			return nil, err
+		}
+	}
 
 	// A little hacky but upgrades the old stats config directives to the new way
 	if result.Telemetry == nil {

command/server/config_test.go

@@ -30,6 +30,14 @@ func TestLoadConfigFile(t *testing.T) {
 			},
 		},
 
+		HABackend: &Backend{
+			Type:          "consul",
+			AdvertiseAddr: "snafu",
+			Config: map[string]string{
+				"bar": "baz",
+			},
+		},
+
 		Telemetry: &Telemetry{
 			StatsdAddr:      "bar",
 			StatsiteAddr:    "foo",
@@ -111,6 +119,13 @@ func TestLoadConfigFile_json2(t *testing.T) {
 			},
 		},
 
+		HABackend: &Backend{
+			Type: "consul",
+			Config: map[string]string{
+				"bar": "baz",
+			},
+		},
+
 		Telemetry: &Telemetry{
 			StatsiteAddr:    "foo",
 			StatsdAddr:      "bar",

command/server/test-fixtures/config.hcl

@@ -12,5 +12,10 @@ backend "consul" {
     advertise_addr = "foo"
 }
 
+ha_backend "consul" {
+    bar = "baz"
+    advertise_addr = "snafu"
+}
+
 max_lease_ttl = "10h"
 default_lease_ttl = "10h"

command/server/test-fixtures/config2.hcl.json

@@ -11,6 +11,12 @@
         }
     },
 
+	"ha_backend": {
+		"consul": {
+			"bar": "baz"
+		}
+	},
+
     "telemetry": {
         "statsd_address": "bar",
         "statsite_address": "foo",

command/server_test.go

@@ -0,0 +1,87 @@
+package command
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/mitchellh/cli"
+)
+
+var (
+	basehcl = `
+disable_mlock = true
+
+listener "tcp" {
+  address = "127.0.0.1:8200"
+  tls_disable = "true"
+}
+`
+
+	consulhcl = `
+backend "consul" {
+    prefix = "foo/"
+	advertise_addr = "http://127.0.0.1:8200"
+}
+`
+	haconsulhcl = `
+ha_backend "consul" {
+    prefix = "bar/"
+	advertise_addr = "http://127.0.0.1:8200"
+}
+`
+
+	badhaconsulhcl = `
+ha_backend "file" {
+    path = "/dev/null"
+}
+`
+)
+
+func TestServer_GoodSeparateHA(t *testing.T) {
+	ui := new(cli.MockUi)
+	c := &ServerCommand{
+		Meta: Meta{
+			Ui: ui,
+		},
+	}
+
+	tmpfile, err := ioutil.TempFile("", "")
+	if err != nil {
+		t.Fatalf("error creating temp dir: %v", err)
+	}
+
+	tmpfile.WriteString(basehcl + consulhcl + haconsulhcl)
+	tmpfile.Close()
+	defer os.Remove(tmpfile.Name())
+
+	args := []string{"-config", tmpfile.Name(), "-verify-only", "true"}
+
+	if code := c.Run(args); code != 0 {
+		t.Fatalf("bad: %d\n\n%s", code, ui.ErrorWriter.String())
+	}
+}
+
+func TestServer_BadSeparateHA(t *testing.T) {
+	ui := new(cli.MockUi)
+	c := &ServerCommand{
+		Meta: Meta{
+			Ui: ui,
+		},
+	}
+
+	tmpfile, err := ioutil.TempFile("", "")
+	if err != nil {
+		t.Fatalf("error creating temp dir: %v", err)
+	}
+
+	tmpfile.WriteString(basehcl + consulhcl + badhaconsulhcl)
+	tmpfile.Close()
+	defer os.Remove(tmpfile.Name())
+
+	args := []string{"-config", tmpfile.Name()}
+
+	if code := c.Run(args); code == 0 {
+		t.Fatalf("bad: should have gotten an error on a bad HA config")
+	}
+}

http/logical_test.go

@@ -69,7 +69,7 @@ func TestLogical_StandbyRedirect(t *testing.T) {
 	defer ln2.Close()
 
 	// Create an HA Vault
-	inm := physical.NewInmemHA()
+	inm := physical.NewInmem()
 	conf := &vault.CoreConfig{
 		Physical:      inm,
 		AdvertiseAddr: addr1,

physical/inmem_ha.go

@@ -32,6 +32,12 @@ func (i *InmemHABackend) LockWith(key, value string) (Lock, error) {
 	return l, nil
 }
 
+// LockMapSize is used in some tests to determine whether this backend has ever
+// been used for HA purposes rather than simply for storage
+func (i *InmemHABackend) LockMapSize() int {
+	return len(i.locks)
+}
+
 // InmemLock is an in-memory Lock implementation for the HABackend
 type InmemLock struct {
 	in    *InmemHABackend

vault/core.go

@@ -273,23 +273,24 @@ type CoreConfig struct {
 	CredentialBackends map[string]logical.Factory
 	AuditBackends      map[string]audit.Factory
 	Physical           physical.Backend
-	Logger             *log.Logger
+
-	DisableCache       bool   // Disables the LRU cache on the physical backend
+	// Defaults to the same backend as Physical. This is not a backend that
-	DisableMlock       bool   // Disables mlock syscall
+	// necessarily supports HA; it is merely the one that will be attempted
-	CacheSize          int    // Custom cache size of zero for default
+	// for HA operations
-	AdvertiseAddr      string // Set as the leader address for HA
+	HAPhysical physical.HABackend
-	DefaultLeaseTTL    time.Duration
+
-	MaxLeaseTTL        time.Duration
+	Logger          *log.Logger
+	DisableCache    bool   // Disables the LRU cache on the physical backend
+	DisableMlock    bool   // Disables mlock syscall
+	CacheSize       int    // Custom cache size of zero for default
+	AdvertiseAddr   string // Set as the leader address for HA
+	DefaultLeaseTTL time.Duration
+	MaxLeaseTTL     time.Duration
 }
 
 // NewCore is used to construct a new core
 func NewCore(conf *CoreConfig) (*Core, error) {
-	// Check if this backend supports an HA configuraiton
+	if conf.HAPhysical != nil && conf.AdvertiseAddr == "" {
-	var haBackend physical.HABackend
-	if ha, ok := conf.Physical.(physical.HABackend); ok {
-		haBackend = ha
-	}
-	if haBackend != nil && conf.AdvertiseAddr == "" {
 		return nil, fmt.Errorf("missing advertisement address")
 	}
 
@@ -299,7 +300,6 @@ func NewCore(conf *CoreConfig) (*Core, error) {
 	if conf.MaxLeaseTTL == 0 {
 		conf.MaxLeaseTTL = maxLeaseTTL
 	}
-
 	if conf.DefaultLeaseTTL > conf.MaxLeaseTTL {
 		return nil, fmt.Errorf("cannot have DefaultLeaseTTL larger than MaxLeaseTTL")
 	}
@@ -355,7 +355,7 @@ func NewCore(conf *CoreConfig) (*Core, error) {
 
 	// Setup the core
 	c := &Core{
-		ha:              haBackend,
+		ha:              conf.HAPhysical,
 		advertiseAddr:   conf.AdvertiseAddr,
 		physical:        conf.Physical,
 		barrier:         barrier,

vault/core_test.go

@@ -1110,10 +1110,12 @@ func TestCore_LimitedUseToken(t *testing.T) {
 
 func TestCore_CleanLeaderPrefix(t *testing.T) {
 	// Create the first core and initialize it
-	inm := physical.NewInmemHA()
+	inm := physical.NewInmem()
+	inmha := physical.NewInmemHA()
 	advertiseOriginal := "http://127.0.0.1:8200"
 	core, err := NewCore(&CoreConfig{
 		Physical:      inm,
+		HAPhysical:    inmha,
 		AdvertiseAddr: advertiseOriginal,
 		DisableMlock:  true,
 	})
@@ -1172,6 +1174,7 @@ func TestCore_CleanLeaderPrefix(t *testing.T) {
 	advertiseOriginal2 := "http://127.0.0.1:8500"
 	core2, err := NewCore(&CoreConfig{
 		Physical:      inm,
+		HAPhysical:    inmha,
 		AdvertiseAddr: advertiseOriginal2,
 		DisableMlock:  true,
 	})
@@ -1255,11 +1258,20 @@ func TestCore_CleanLeaderPrefix(t *testing.T) {
 }
 
 func TestCore_Standby(t *testing.T) {
+	inmha := physical.NewInmemHA()
+	testCore_Standby_Common(t, inmha, inmha)
+}
+
+func TestCore_Standby_SeparateHA(t *testing.T) {
+	testCore_Standby_Common(t, physical.NewInmemHA(), physical.NewInmemHA())
+}
+
+func testCore_Standby_Common(t *testing.T, inm physical.Backend, inmha physical.HABackend) {
 	// Create the first core and initialize it
-	inm := physical.NewInmemHA()
 	advertiseOriginal := "http://127.0.0.1:8200"
 	core, err := NewCore(&CoreConfig{
 		Physical:      inm,
+		HAPhysical:    inmha,
 		AdvertiseAddr: advertiseOriginal,
 		DisableMlock:  true,
 	})
@@ -1313,6 +1325,7 @@ func TestCore_Standby(t *testing.T) {
 	advertiseOriginal2 := "http://127.0.0.1:8500"
 	core2, err := NewCore(&CoreConfig{
 		Physical:      inm,
+		HAPhysical:    inmha,
 		AdvertiseAddr: advertiseOriginal2,
 		DisableMlock:  true,
 	})
@@ -1404,6 +1417,23 @@ func TestCore_Standby(t *testing.T) {
 	if advertise != advertiseOriginal2 {
 		t.Fatalf("Bad advertise: %v", advertise)
 	}
+
+	if inm.(*physical.InmemHABackend) == inmha.(*physical.InmemHABackend) {
+		lockSize := inm.(*physical.InmemHABackend).LockMapSize()
+		if lockSize == 0 {
+			t.Fatalf("locks not used with only one HA backend")
+		}
+	} else {
+		lockSize := inmha.(*physical.InmemHABackend).LockMapSize()
+		if lockSize == 0 {
+			t.Fatalf("locks not used with expected HA backend")
+		}
+
+		lockSize = inm.(*physical.InmemHABackend).LockMapSize()
+		if lockSize != 0 {
+			t.Fatalf("locks used with unexpected HA backend")
+		}
+	}
 }
 
 // Ensure that InternalData is never returned
@@ -2003,10 +2033,12 @@ func testWaitActive(t *testing.T, core *Core) {
 
 func TestCore_Standby_Rotate(t *testing.T) {
 	// Create the first core and initialize it
-	inm := physical.NewInmemHA()
+	inm := physical.NewInmem()
+	inmha := physical.NewInmemHA()
 	advertiseOriginal := "http://127.0.0.1:8200"
 	core, err := NewCore(&CoreConfig{
 		Physical:      inm,
+		HAPhysical:    inmha,
 		AdvertiseAddr: advertiseOriginal,
 		DisableMlock:  true,
 	})
@@ -2025,6 +2057,7 @@ func TestCore_Standby_Rotate(t *testing.T) {
 	advertiseOriginal2 := "http://127.0.0.1:8500"
 	core2, err := NewCore(&CoreConfig{
 		Physical:      inm,
+		HAPhysical:    inmha,
 		AdvertiseAddr: advertiseOriginal2,
 		DisableMlock:  true,
 	})
@@ -2074,10 +2107,12 @@ func TestCore_Standby_Rotate(t *testing.T) {
 
 func TestCore_Standby_Rekey(t *testing.T) {
 	// Create the first core and initialize it
-	inm := physical.NewInmemHA()
+	inm := physical.NewInmem()
+	inmha := physical.NewInmemHA()
 	advertiseOriginal := "http://127.0.0.1:8200"
 	core, err := NewCore(&CoreConfig{
 		Physical:      inm,
+		HAPhysical:    inmha,
 		AdvertiseAddr: advertiseOriginal,
 		DisableMlock:  true,
 	})
@@ -2096,6 +2131,7 @@ func TestCore_Standby_Rekey(t *testing.T) {
 	advertiseOriginal2 := "http://127.0.0.1:8500"
 	core2, err := NewCore(&CoreConfig{
 		Physical:      inm,
+		HAPhysical:    inmha,
 		AdvertiseAddr: advertiseOriginal2,
 		DisableMlock:  true,
 	})