mirrors/vault
1
0
Fork 0
mirror of https://github.com/hashicorp/vault.git synced 2026-05-05 12:26:34 +02:00
Code Issues Projects Releases Wiki Activity

Add DR failover scenario to Enos (#28256)

Browse Source 
* Add DR failover scenario to Enos

* Update enos/enos-scenario-dr-replication.hcl

Co-authored-by: Ryan Cragun <me@ryan.ec>

* Update enos/enos-qualities.hcl

Co-authored-by: Ryan Cragun <me@ryan.ec>

* Update enos/enos-scenario-dr-replication.hcl

Co-authored-by: Ryan Cragun <me@ryan.ec>

* Update enos/enos-scenario-dr-replication.hcl

Co-authored-by: Ryan Cragun <me@ryan.ec>

* Update enos/enos-qualities.hcl

Co-authored-by: Ryan Cragun <me@ryan.ec>

* Update enos/enos-scenario-dr-replication.hcl

Co-authored-by: Ryan Cragun <me@ryan.ec>

* Update enos/enos-scenario-dr-replication.hcl

Co-authored-by: Ryan Cragun <me@ryan.ec>

* Update enos/enos-scenario-dr-replication.hcl

Co-authored-by: Ryan Cragun <me@ryan.ec>

* Update enos/enos-scenario-pr-replication.hcl

Co-authored-by: Ryan Cragun <me@ryan.ec>

* remove superuser

* Update enos/enos-scenario-dr-replication.hcl

Co-authored-by: Ryan Cragun <me@ryan.ec>

* Update enos/enos-scenario-dr-replication.hcl

Co-authored-by: Ryan Cragun <me@ryan.ec>

* Update enos/enos-scenario-dr-replication.hcl

Co-authored-by: Ryan Cragun <me@ryan.ec>

* Update enos/enos-scenario-dr-replication.hcl

Co-authored-by: Ryan Cragun <me@ryan.ec>

* Update enos/enos-scenario-dr-replication.hcl

Co-authored-by: Ryan Cragun <me@ryan.ec>

---------

Co-authored-by: Ryan Cragun <me@ryan.ec>
This commit is contained in:
Luis (LT) Carbonell 2024-09-05 17:33:53 -04:00 committed by GitHub
parent 3bda80649f
commit cdf3da4066
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
21 changed files with 2336 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
enos/enos-dev-scenario-pr-replication.hcl
View File

@ -756,10 +756,12 @@ scenario "dev_pr_replication" {
}
variables {
primary_leader_public_ip = step.get_primary_cluster_ips.leader_public_ip
vault_addr = step.create_primary_cluster.api_addr_localhost
vault_install_dir = local.vault_install_dir
vault_root_token = step.create_primary_cluster.root_token
ip_version = local.ip_version
primary_leader_host = step.get_primary_cluster_ips.leader_host
replication_type = "performance"
vault_addr = step.create_primary_cluster.api_addr_localhost
vault_install_dir = local.vault_install_dir
vault_root_token = step.create_primary_cluster.root_token
}
}
@ -776,10 +778,12 @@ scenario "dev_pr_replication" {
}
variables {
primary_leader_public_ip = step.get_primary_cluster_ips.leader_public_ip
vault_addr = step.create_primary_cluster.api_addr_localhost
vault_install_dir = local.vault_install_dir
vault_root_token = step.create_primary_cluster.root_token
ip_version = local.ip_version
primary_leader_host = step.get_primary_cluster_ips.leader_host
replication_type = "performance"
vault_addr = step.create_primary_cluster.api_addr_localhost
vault_install_dir = local.vault_install_dir
vault_root_token = step.create_primary_cluster.root_token
}
}
@ -787,7 +791,7 @@ scenario "dev_pr_replication" {
description = <<-EOF
Enable performance replication on the secondary using the new shared token.
EOF
module = module.vault_setup_perf_secondary
module = module.vault_setup_replication_secondary
depends_on = [step.generate_secondary_token]
providers = {
@ -795,11 +799,13 @@ scenario "dev_pr_replication" {
}
variables {
secondary_leader_public_ip = step.get_secondary_cluster_ips.leader_public_ip
vault_addr = step.create_secondary_cluster.api_addr_localhost
vault_install_dir = local.vault_install_dir
vault_root_token = step.create_secondary_cluster.root_token
wrapping_token = step.generate_secondary_token.secondary_token
ip_version = local.ip_version
secondary_leader_host = step.get_secondary_cluster_ips.leader_host
replication_type = "performance"
vault_addr = step.create_secondary_cluster.api_addr_localhost
vault_install_dir = local.vault_install_dir
vault_root_token = step.create_secondary_cluster.root_token
wrapping_token = step.generate_secondary_token.secondary_token
}
}

97
enos/enos-modules.hcl
View File

@ -56,6 +56,24 @@ module "get_local_metadata" {
source = "./modules/get_local_metadata"
}
module "generate_dr_operation_token" {
source = "./modules/generate_dr_operation_token"
vault_install_dir = var.vault_install_dir
}
module "generate_failover_secondary_token" {
source = "./modules/generate_failover_secondary_token"
vault_install_dir = var.vault_install_dir
}
module "generate_secondary_public_key" {
source = "./modules/generate_secondary_public_key"
vault_install_dir = var.vault_install_dir
}
module "generate_secondary_token" {
source = "./modules/generate_secondary_token"
@ -185,13 +203,43 @@ module "vault_get_cluster_ips" {
vault_install_dir = var.vault_install_dir
}
module "vault_failover_demote_dr_primary" {
source = "./modules/vault_failover_demote_dr_primary"
vault_install_dir = var.vault_install_dir
}
module "vault_failover_promote_dr_secondary" {
source = "./modules/vault_failover_promote_dr_secondary"
vault_install_dir = var.vault_install_dir
}
module "vault_failover_update_dr_primary" {
source = "./modules/vault_failover_update_dr_primary"
vault_install_dir = var.vault_install_dir
}
module "vault_raft_remove_peer" {
source = "./modules/vault_raft_remove_peer"
vault_install_dir = var.vault_install_dir
}
module "vault_setup_perf_secondary" {
source = "./modules/vault_setup_perf_secondary"
module "vault_setup_dr_primary" {
source = "./modules/vault_setup_dr_primary"
vault_install_dir = var.vault_install_dir
}
module "vault_setup_perf_primary" {
source = "./modules/vault_setup_perf_primary"
vault_install_dir = var.vault_install_dir
}
module "vault_setup_replication_secondary" {
source = "./modules/vault_setup_replication_secondary"
vault_install_dir = var.vault_install_dir
}
@ -227,6 +275,12 @@ module "vault_verify_autopilot" {
vault_install_dir = var.vault_install_dir
}
module "vault_verify_dr_replication" {
source = "./modules/vault_verify_dr_replication"
vault_install_dir = var.vault_install_dir
}
module "vault_verify_raft_auto_join_voter" {
source = "./modules/vault_verify_raft_auto_join_voter"
@ -234,11 +288,6 @@ module "vault_verify_raft_auto_join_voter" {
vault_cluster_addr_port = global.ports["vault_cluster"]["port"]
}
module "vault_verify_undo_logs" {
source = "./modules/vault_verify_undo_logs"
vault_install_dir = var.vault_install_dir
}
module "vault_verify_default_lcq" {
source = "./modules/vault_verify_default_lcq"
@ -250,22 +299,6 @@ module "vault_verify_replication" {
source = "./modules/vault_verify_replication"
}
module "vault_verify_ui" {
source = "./modules/vault_verify_ui"
}
module "vault_verify_unsealed" {
source = "./modules/vault_verify_unsealed"
vault_install_dir = var.vault_install_dir
}
module "vault_setup_perf_primary" {
source = "./modules/vault_setup_perf_primary"
vault_install_dir = var.vault_install_dir
}
module "vault_verify_read_data" {
source = "./modules/vault_verify_read_data"
@ -290,6 +323,22 @@ module "vault_verify_write_data" {
vault_install_dir = var.vault_install_dir
}
module "vault_verify_ui" {
source = "./modules/vault_verify_ui"
}
module "vault_verify_undo_logs" {
source = "./modules/vault_verify_undo_logs"
vault_install_dir = var.vault_install_dir
}
module "vault_verify_unsealed" {
source = "./modules/vault_verify_unsealed"
vault_install_dir = var.vault_install_dir
}
module "vault_wait_for_leader" {
source = "./modules/vault_wait_for_leader"
@ -314,4 +363,4 @@ module "vault_verify_billing_start_date" {
vault_install_dir = var.vault_install_dir
vault_instance_count = var.vault_instance_count
vault_cluster_addr_port = global.ports["vault_cluster"]["port"]
}
}

53
enos/enos-qualities.hcl
View File

@ -120,6 +120,59 @@ quality "vault_api_sys_quotas_lease_count_read_max_leases_default" {
EOF
}
quality "vault_api_sys_replication_dr_primary_enable_write" {
description = <<-EOF
The v1/sys/replication/dr/primary/enable Vault API enables DR replication
EOF
}
quality "vault_api_sys_replication_dr_primary_secondary_token_write" {
description = <<-EOF
The v1/sys/replication/dr/primary/secondary-token Vault API configures the DR replication
secondary token
EOF
}
quality "vault_api_sys_replication_dr_secondary_enable_write" {
description = <<-EOF
The v1/sys/replication/dr/secondary/enable Vault API enables DR replication
EOF
}
quality "vault_api_sys_replication_dr_read_connection_status_connected" {
description = <<-EOF
The v1/sys/replication/dr/status Vault API returns status info and the
'connection_status' is correct for the given node
EOF
}
quality "vault_api_sys_replication_dr_status_known_primary_cluster_addrs" {
description = <<-EOF
The v1/sys/replication/dr/status Vault API returns the DR replication status and
'known_primary_cluster_address' is the expected primary cluster leader
EOF
}
quality "vault_api_sys_replication_dr_status_read" {
description = <<-EOF
The v1/sys/replication/dr/status Vault API returns the DR replication status
EOF
}
quality "vault_api_sys_replication_dr_status_read_cluster_address" {
description = <<-EOF
The v1/sys/replication/dr/status Vault API returns the DR replication status
and the '{primaries,secondaries}[*].cluster_address' is correct for the given node
EOF
}
quality "vault_api_sys_replication_dr_status_read_state_not_idle" {
description = <<-EOF
The v1/sys/replication/dr/status Vault API returns the DR replication status
and the state is not idle
EOF
}
quality "vault_api_sys_replication_performance_primary_enable_write" {
description = <<-EOF
The v1/sys/replication/performance/primary/enable Vault API enables performance replication

1258
enos/enos-scenario-dr-replication.hcl Normal file
View File

File diff suppressed because it is too large Load Diff

39
enos/enos-scenario-replication.hcl → enos/enos-scenario-pr-replication.hcl
View File

@ -1,9 +1,9 @@
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: BUSL-1.1
scenario "replication" {
scenario "pr_replication" {
description = <<-EOF
The replication scenario configures performance replication between two Vault clusters and
The PR replication scenario configures performance replication between two Vault clusters and
verifies behavior and failure tolerance. The build can be a local branch, any CRT built Vault
Enterprise artifact saved to the local machine, or any CRT built Vault Enterprise artifact in
the stable channel in Artifactory.
@ -704,7 +704,7 @@ scenario "replication" {
step "configure_performance_replication_primary" {
description = <<-EOF
Create the necessary superuser auth policy necessary for performance replicaztion, assign it
Create the necessary superuser auth policy necessary for performance replication, assign it
to a our previously create test user, and enable performance replication on the primary
cluster.
EOF
@ -731,10 +731,11 @@ scenario "replication" {
]
variables {
primary_leader_public_ip = step.get_primary_cluster_ips.leader_public_ip
vault_addr = step.create_primary_cluster.api_addr_localhost
vault_install_dir = global.vault_install_dir[matrix.artifact_type]
vault_root_token = step.create_primary_cluster.root_token
ip_version = matrix.ip_version
primary_leader_host = step.get_primary_cluster_ips.leader_host
vault_addr = step.create_primary_cluster.api_addr_localhost
vault_install_dir = global.vault_install_dir[matrix.artifact_type]
vault_root_token = step.create_primary_cluster.root_token
}
}
@ -754,10 +755,12 @@ scenario "replication" {
}
variables {
primary_leader_public_ip = step.get_primary_cluster_ips.leader_public_ip
vault_addr = step.create_primary_cluster.api_addr_localhost
vault_install_dir = global.vault_install_dir[matrix.artifact_type]
vault_root_token = step.create_primary_cluster.root_token
ip_version = matrix.ip_version
primary_leader_host = step.get_primary_cluster_ips.leader_host
replication_type = "performance"
vault_addr = step.create_primary_cluster.api_addr_localhost
vault_install_dir = global.vault_install_dir[matrix.artifact_type]
vault_root_token = step.create_primary_cluster.root_token
}
}
@ -766,7 +769,7 @@ scenario "replication" {
Enable performance replication on the secondary cluster with the wrapping token created by
the primary cluster.
EOF
module = module.vault_setup_perf_secondary
module = module.vault_setup_replication_secondary
depends_on = [step.generate_secondary_token]
providers = {
@ -776,11 +779,13 @@ scenario "replication" {
verifies = quality.vault_api_sys_replication_performance_secondary_enable_write
variables {
secondary_leader_public_ip = step.get_secondary_cluster_ips.leader_public_ip
vault_addr = step.create_secondary_cluster.api_addr_localhost
vault_install_dir = global.vault_install_dir[matrix.artifact_type]
vault_root_token = step.create_secondary_cluster.root_token
wrapping_token = step.generate_secondary_token.secondary_token
ip_version = matrix.ip_version
secondary_leader_host = step.get_secondary_cluster_ips.leader_host
replication_type = "performance"
vault_addr = step.create_secondary_cluster.api_addr_localhost
vault_install_dir = global.vault_install_dir[matrix.artifact_type]
vault_root_token = step.create_secondary_cluster.root_token
wrapping_token = step.generate_secondary_token.secondary_token
}
}

82
enos/modules/generate_dr_operation_token/main.tf Normal file
View File

@ -0,0 +1,82 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: BUSL-1.1
terraform {
required_providers {
enos = {
source = "registry.terraform.io/hashicorp-forge/enos"
}
random = {
source = "hashicorp/random"
version = ">= 3.4.3"
}
}
}
variable "ip_version" {
type = number
description = "The IP version used for the Vault TCP listener"
validation {
condition = contains([4, 6], var.ip_version)
error_message = "The ip_version must be either 4 or 6"
}
}
variable "primary_leader_host" {
type = object({
ipv6 = string
private_ip = string
public_ip = string
})
description = "The primary cluster leader host"
}
variable "vault_addr" {
type = string
description = "The local vault API listen address"
}
variable "vault_install_dir" {
type = string
description = "The directory where the Vault binary will be installed"
}
variable "vault_root_token" {
type = string
description = "The vault root token"
}
variable "storage_backend" {
type = string
description = "The storage backend to use for the Vault cluster"
}
locals {
token_id = random_uuid.token_id.id
dr_operation_token = enos_remote_exec.fetch_dr_operation_token.stdout
}
resource "random_uuid" "token_id" {}
resource "enos_remote_exec" "fetch_dr_operation_token" {
depends_on = [random_uuid.token_id]
environment = {
VAULT_ADDR = var.vault_addr
VAULT_INSTALL_DIR = var.vault_install_dir
VAULT_TOKEN = var.vault_root_token
STORAGE_BACKEND = var.storage_backend
}
scripts = [abspath("${path.module}/scripts/configure-vault-dr-primary.sh")]
transport = {
ssh = {
host = var.primary_leader_host.public_ip
}
}
}
output "dr_operation_token" {
value = local.dr_operation_token
}

50
enos/modules/generate_dr_operation_token/scripts/configure-vault-dr-primary.sh Executable file
View File

@ -0,0 +1,50 @@
#!/usr/bin/env bash
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: BUSL-1.1
set -e
binpath="${VAULT_INSTALL_DIR}/vault"
fail() {
echo "$1" >&2
exit 1
}
# Check required environment variables
[[ -z "$VAULT_ADDR" ]] && fail "VAULT_ADDR env variable has not been set"
[[ -z "$VAULT_INSTALL_DIR" ]] && fail "VAULT_INSTALL_DIR env variable has not been set"
[[ -z "$STORAGE_BACKEND" ]] && fail "STORAGE_BACKEND env variable has not been set"
# Define the policy content
policy_content() {
cat << EOF
path "sys/replication/dr/secondary/promote" {
capabilities = [ "update" ]
}
path "sys/replication/dr/secondary/update-primary" {
capabilities = [ "update" ]
}
EOF
if [ "$STORAGE_BACKEND" = "raft" ]; then
cat << EOF
path "sys/storage/raft/autopilot/state" {
capabilities = [ "update", "read" ]
}
EOF
fi
}
# Write the policy
$binpath policy write dr-secondary-promotion - <<< "$(policy_content)" &> /dev/null
# Configure the failover handler token role
$binpath write auth/token/roles/failover-handler \
allowed_policies=dr-secondary-promotion \
orphan=true \
renewable=false \
token_type=batch &> /dev/null
# Create a token for the failover handler role and output the token only
$binpath token create -field=token -role=failover-handler -ttl=8h

98
enos/modules/generate_failover_secondary_token/main.tf Normal file
View File

@ -0,0 +1,98 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: BUSL-1.1
terraform {
required_providers {
enos = {
source = "registry.terraform.io/hashicorp-forge/enos"
}
random = {
source = "hashicorp/random"
version = ">= 3.4.3"
}
}
}
variable "ip_version" {
type = number
description = "The IP version used for the Vault TCP listener"
validation {
condition = contains([4, 6], var.ip_version)
error_message = "The ip_version must be either 4 or 6"
}
}
variable "primary_leader_host" {
type = object({
ipv6 = string
private_ip = string
public_ip = string
})
description = "The primary cluster leader host"
}
variable "retry_interval" {
type = string
default = "2"
description = "How long to wait between retries"
}
variable "secondary_public_key" {
type = string
description = "The secondary public key"
}
variable "timeout" {
type = string
default = "15"
description = "How many seconds to wait before timing out"
}
variable "vault_addr" {
type = string
description = "The local vault API listen address"
}
variable "vault_install_dir" {
type = string
description = "The directory where the Vault binary will be installed"
}
variable "vault_root_token" {
type = string
description = "The vault root token"
}
locals {
primary_leader_addr = var.ip_version == 6 ? var.primary_leader_host.ipv6 : var.primary_leader_host.private_ip
token_id = random_uuid.token_id.id
secondary_token = enos_remote_exec.fetch_secondary_token.stdout
}
resource "random_uuid" "token_id" {}
resource "enos_remote_exec" "fetch_secondary_token" {
depends_on = [random_uuid.token_id]
environment = {
VAULT_ADDR = var.vault_addr
VAULT_TOKEN = var.vault_root_token
RETRY_INTERVAL = var.retry_interval
TIMEOUT_SECONDS = var.timeout
SECONDARY_PUBLIC_KEY = var.secondary_public_key
VAULT_INSTALL_DIR = var.vault_install_dir
}
scripts = [abspath("${path.module}/scripts/generate-failover-secondary-token.sh")]
transport = {
ssh = {
host = var.primary_leader_host.public_ip
}
}
}
output "secondary_token" {
value = local.secondary_token
}

33
enos/modules/generate_failover_secondary_token/scripts/generate-failover-secondary-token.sh Normal file
View File

@ -0,0 +1,33 @@
#!/bin/bash
## Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: BUSL-1.1
set -e
[[ -z "${VAULT_INSTALL_DIR}" ]] && fail "VAULT_INSTALL_DIR env variable has not been set"
[[ -z "${RETRY_INTERVAL}" ]] && fail "RETRY_INTERVAL env variable has not been set"
[[ -z "${TIMEOUT_SECONDS}" ]] && fail "TIMEOUT_SECONDS env variable has not been set"
[[ -z "${VAULT_ADDR}" ]] && fail "VAULT_ADDR env variable has not been set"
[[ -z "${VAULT_TOKEN}" ]] && fail "VAULT_TOKEN env variable has not been set"
[[ -z "${SECONDARY_PUBLIC_KEY}" ]] && fail "SECONDARY_PUBLIC_KEY env variable has not been set"
fail() {
echo "$1" 1>&2
exit 1
}
binpath="${VAULT_INSTALL_DIR}"/vault
test -x "${binpath}" || fail "unable to locate vault binary at ${binpath}"
begin_time=$(date +%s)
end_time=$((begin_time + TIMEOUT_SECONDS))
while [ "$(date +%s)" -lt "${end_time}" ]; do
if secondary_token=$(${binpath} write -field token sys/replication/dr/primary/secondary-token id="${VAULT_TOKEN}" secondary_public_key="${SECONDARY_PUBLIC_KEY}"); then
echo "${secondary_token}"
exit 0
fi
sleep "${RETRY_INTERVAL}"
done
fail "Timed out trying to generate secondary token"

77
enos/modules/generate_secondary_public_key/main.tf Normal file
View File

@ -0,0 +1,77 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: BUSL-1.1
terraform {
required_providers {
enos = {
source = "registry.terraform.io/hashicorp-forge/enos"
}
random = {
source = "hashicorp/random"
version = ">= 3.4.3"
}
}
}
variable "ip_version" {
type = number
description = "The IP version used for the Vault TCP listener"
validation {
condition = contains([4, 6], var.ip_version)
error_message = "The ip_version must be either 4 or 6"
}
}
variable "primary_leader_host" {
type = object({
ipv6 = string
private_ip = string
public_ip = string
})
description = "The primary cluster leader host"
}
variable "vault_addr" {
type = string
description = "The local vault API listen address"
}
variable "vault_install_dir" {
type = string
description = "The directory where the Vault binary will be installed"
}
variable "vault_root_token" {
type = string
description = "The vault root token"
}
locals {
primary_leader_addr = var.ip_version == 6 ? var.primary_leader_host.ipv6 : var.primary_leader_host.private_ip
token_id = random_uuid.token_id.id
secondary_public_key = enos_remote_exec.fetch_secondary_public_key.stdout
}
resource "random_uuid" "token_id" {}
resource "enos_remote_exec" "fetch_secondary_public_key" {
depends_on = [random_uuid.token_id]
environment = {
VAULT_ADDR = var.vault_addr
VAULT_TOKEN = var.vault_root_token
}
inline = ["${var.vault_install_dir}/vault write -field secondary_public_key -f sys/replication/dr/secondary/generate-public-key"]
transport = {
ssh = {
host = var.primary_leader_host.public_ip
}
}
}
output "secondary_public_key" {
value = local.secondary_public_key
}

43
enos/modules/generate_secondary_token/main.tf
View File

@ -13,6 +13,35 @@ terraform {
}
}
variable "ip_version" {
type = number
description = "The IP version used for the Vault TCP listener"
validation {
condition = contains([4, 6], var.ip_version)
error_message = "The ip_version must be either 4 or 6"
}
}
variable "primary_leader_host" {
type = object({
ipv6 = string
private_ip = string
public_ip = string
})
description = "The primary cluster leader host"
}
variable "replication_type" {
type = string
description = "The type of replication to perform"
validation {
condition = contains(["dr", "performance"], var.replication_type)
error_message = "The replication_type must be either dr or performance"
}
}
variable "vault_addr" {
type = string
description = "The local vault API listen address"
@ -23,19 +52,15 @@ variable "vault_install_dir" {
description = "The directory where the Vault binary will be installed"
}
variable "primary_leader_public_ip" {
type = string
description = "Vault primary cluster leader Public IP address"
}
variable "vault_root_token" {
type = string
description = "The vault root token"
}
locals {
token_id = random_uuid.token_id.id
secondary_token = enos_remote_exec.fetch_secondary_token.stdout
primary_leader_addr = var.ip_version == 6 ? var.primary_leader_host.ipv6 : var.primary_leader_host.private_ip
token_id = random_uuid.token_id.id
secondary_token = enos_remote_exec.fetch_secondary_token.stdout
}
resource "random_uuid" "token_id" {}
@ -47,11 +72,11 @@ resource "enos_remote_exec" "fetch_secondary_token" {
VAULT_TOKEN = var.vault_root_token
}
inline = ["${var.vault_install_dir}/vault write sys/replication/performance/primary/secondary-token id=${local.token_id} |sed -n '/^wrapping_token:/p' |awk '{print $2}'"]
inline = ["${var.vault_install_dir}/vault write sys/replication/${var.replication_type}/primary/secondary-token id=${local.token_id} |sed -n '/^wrapping_token:/p' |awk '{print $2}'"]
transport = {
ssh = {
host = var.primary_leader_public_ip
host = var.primary_leader_host.public_ip
}
}
}

63
enos/modules/vault_failover_demote_dr_primary/main.tf Normal file
View File

@ -0,0 +1,63 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: BUSL-1.1
terraform {
required_providers {
enos = {
source = "registry.terraform.io/hashicorp-forge/enos"
}
}
}
variable "ip_version" {
type = number
description = "The IP version used for the Vault TCP listener"
validation {
condition = contains([4, 6], var.ip_version)
error_message = "The ip_version must be either 4 or 6"
}
}
variable "primary_leader_host" {
type = object({
ipv6 = string
private_ip = string
public_ip = string
})
description = "The primary cluster leader host"
}
variable "vault_install_dir" {
type = string
description = "The directory where the Vault binary will be installed"
}
variable "vault_addr" {
type = string
description = "The local vault API listen address"
}
variable "vault_root_token" {
type = string
description = "The vault root token"
}
locals {
}
resource "enos_remote_exec" "demote_dr_primary" {
environment = {
VAULT_ADDR = var.vault_addr
VAULT_TOKEN = var.vault_root_token
}
inline = ["${var.vault_install_dir}/vault write -f sys/replication/dr/primary/demote"]
transport = {
ssh = {
host = var.primary_leader_host.public_ip
}
}
}

69
enos/modules/vault_failover_promote_dr_secondary/main.tf Normal file
View File

@ -0,0 +1,69 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: BUSL-1.1
terraform {
required_providers {
enos = {
source = "registry.terraform.io/hashicorp-forge/enos"
}
}
}
variable "ip_version" {
type = number
description = "The IP version used for the Vault TCP listener"
validation {
condition = contains([4, 6], var.ip_version)
error_message = "The ip_version must be either 4 or 6"
}
}
variable "secondary_leader_host" {
type = object({
ipv6 = string
private_ip = string
public_ip = string
})
description = "The secondary cluster leader host"
}
variable "vault_addr" {
type = string
description = "The local vault API listen address"
}
variable "vault_install_dir" {
type = string
description = "The directory where the Vault binary will be installed"
}
variable "vault_root_token" {
type = string
description = "The vault root token"
}
variable "dr_operation_token" {
type = string
description = "The wrapping token created on primary cluster"
}
locals {
dr_operation_token = var.dr_operation_token
}
resource "enos_remote_exec" "promote_dr_secondary" {
environment = {
VAULT_ADDR = var.vault_addr
VAULT_TOKEN = var.vault_root_token
}
inline = ["${var.vault_install_dir}/vault write -f sys/replication/dr/secondary/promote dr_operation_token=${local.dr_operation_token}"]
transport = {
ssh = {
host = var.secondary_leader_host.public_ip
}
}
}

76
enos/modules/vault_failover_update_dr_primary/main.tf Normal file
View File

@ -0,0 +1,76 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: BUSL-1.1
terraform {
required_providers {
enos = {
source = "registry.terraform.io/hashicorp-forge/enos"
}
}
}
variable "ip_version" {
type = number
description = "The IP version used for the Vault TCP listener"
validation {
condition = contains([4, 6], var.ip_version)
error_message = "The ip_version must be either 4 or 6"
}
}
variable "secondary_leader_host" {
type = object({
ipv6 = string
private_ip = string
public_ip = string
})
description = "The secondary cluster leader host"
}
variable "vault_install_dir" {
type = string
description = "The directory where the Vault binary will be installed"
}
variable "vault_addr" {
type = string
description = "The local vault API listen address"
}
variable "vault_root_token" {
type = string
description = "The vault root token"
}
variable "dr_operation_token" {
type = string
description = "The wrapping token created on primary cluster"
}
variable "wrapping_token" {
type = string
description = "The wrapping token created on primary cluster"
}
locals {
dr_operation_token = var.dr_operation_token
wrapping_token = var.wrapping_token
}
resource "enos_remote_exec" "update_dr_primary" {
environment = {
VAULT_ADDR = var.vault_addr
VAULT_TOKEN = var.vault_root_token
}
inline = ["${var.vault_install_dir}/vault write sys/replication/dr/secondary/update-primary dr_operation_token=${local.dr_operation_token} token=${local.wrapping_token}"]
transport = {
ssh = {
host = var.secondary_leader_host.public_ip
}
}
}

59
enos/modules/vault_setup_dr_primary/main.tf Normal file
View File

@ -0,0 +1,59 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: BUSL-1.1
terraform {
required_providers {
enos = {
source = "registry.terraform.io/hashicorp-forge/enos"
}
}
}
variable "ip_version" {
type = number
description = "The IP version used for the Vault TCP listener"
validation {
condition = contains([4, 6], var.ip_version)
error_message = "The ip_version must be either 4 or 6"
}
}
variable "primary_leader_host" {
type = object({
ipv6 = string
private_ip = string
public_ip = string
})
description = "The primary cluster leader host"
}
variable "vault_addr" {
type = string
description = "The local vault API listen address"
}
variable "vault_install_dir" {
type = string
description = "The directory where the Vault binary will be installed"
}
variable "vault_root_token" {
type = string
description = "The vault root token"
}
resource "enos_remote_exec" "configure_dr_primary" {
environment = {
VAULT_ADDR = var.vault_addr
VAULT_TOKEN = var.vault_root_token
VAULT_INSTALL_DIR = var.vault_install_dir
}
scripts = [abspath("${path.module}/scripts/configure-vault-dr-primary.sh")]
transport = {
ssh = {
host = var.primary_leader_host.public_ip
}
}
}

17
enos/modules/vault_setup_dr_primary/scripts/configure-vault-dr-primary.sh Normal file
View File

@ -0,0 +1,17 @@
#!/usr/bin/env bash
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: BUSL-1.1
set -e
binpath=${VAULT_INSTALL_DIR}/vault
fail() {
echo "$1" 1>&2
return 1
}
test -x "$binpath" || fail "unable to locate vault binary at $binpath"
# Activate the primary
$binpath write -f sys/replication/dr/primary/enable

22
enos/modules/vault_setup_perf_primary/main.tf
View File

@ -9,9 +9,23 @@ terraform {
}
}
variable "primary_leader_public_ip" {
type = string
description = "Vault primary cluster leader Public IP address"
variable "ip_version" {
type = number
description = "The IP version used for the Vault TCP listener"
validation {
condition = contains([4, 6], var.ip_version)
error_message = "The ip_version must be either 4 or 6"
}
}
variable "primary_leader_host" {
type = object({
ipv6 = string
private_ip = string
public_ip = string
})
description = "The primary cluster leader host"
}
variable "vault_addr" {
@ -40,7 +54,7 @@ resource "enos_remote_exec" "configure_pr_primary" {
transport = {
ssh = {
host = var.primary_leader_public_ip
host = var.primary_leader_host.public_ip
}
}
}

12
enos/modules/vault_setup_perf_primary/scripts/configure-vault-pr-primary.sh
View File

@ -13,17 +13,5 @@ fail() {
test -x "$binpath" || fail "unable to locate vault binary at $binpath"
# Create superuser policy
$binpath policy write superuser - << EOF
path "*" {
capabilities = ["create", "read", "update", "delete", "list", "sudo"]
}
EOF
# The userpass auth method is enabled with the `vault_verify_write_data`,
# so we do not enable here.
# Create new user and attach superuser policy
$binpath write auth/userpass/users/tester password="changeme" policies="superuser"
# Activate the primary
$binpath write -f sys/replication/performance/primary/enable

32
enos/modules/vault_setup_perf_secondary/main.tf → enos/modules/vault_setup_replication_secondary/main.tf
View File

@ -9,9 +9,33 @@ terraform {
}
}
variable "secondary_leader_public_ip" {
variable "ip_version" {
type = number
description = "The IP version used for the Vault TCP listener"
validation {
condition = contains([4, 6], var.ip_version)
error_message = "The ip_version must be either 4 or 6"
}
}
variable "secondary_leader_host" {
type = object({
ipv6 = string
private_ip = string
public_ip = string
})
description = "The secondary cluster leader host"
}
variable "replication_type" {
type = string
description = "Vault secondary cluster leader Public IP address"
description = "The type of replication to perform"
validation {
condition = contains(["dr", "performance"], var.replication_type)
error_message = "The replication_type must be either dr or performance"
}
}
variable "vault_addr" {
@ -40,11 +64,11 @@ resource "enos_remote_exec" "configure_pr_secondary" {
VAULT_TOKEN = var.vault_root_token
}
inline = ["${var.vault_install_dir}/vault write sys/replication/performance/secondary/enable token=${var.wrapping_token}"]
inline = ["${var.vault_install_dir}/vault write sys/replication/${var.replication_type}/secondary/enable token=${var.wrapping_token}"]
transport = {
ssh = {
host = var.secondary_leader_public_ip
host = var.secondary_leader_host.public_ip
}
}
}

117
enos/modules/vault_verify_dr_replication/main.tf Normal file
View File

@ -0,0 +1,117 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: BUSL-1.1
terraform {
required_providers {
enos = {
source = "registry.terraform.io/hashicorp-forge/enos"
}
}
}
variable "ip_version" {
type = number
description = "The IP version used for the Vault TCP listener"
validation {
condition = contains([4, 6], var.ip_version)
error_message = "The ip_version must be either 4 or 6"
}
}
variable "primary_leader_host" {
type = object({
ipv6 = string
private_ip = string
public_ip = string
})
description = "The primary cluster leader host"
}
variable "secondary_leader_host" {
type = object({
ipv6 = string
private_ip = string
public_ip = string
})
description = "The secondary cluster leader host"
}
variable "vault_addr" {
type = string
description = "The local vault API listen address"
}
variable "vault_install_dir" {
type = string
description = "The directory where the Vault binary will be installed"
}
variable "wrapping_token" {
type = string
description = "The wrapping token created on primary cluster"
default = null
}
locals {
primary_leader_addr = var.ip_version == 6 ? var.primary_leader_host.ipv6 : var.primary_leader_host.private_ip
secondary_leader_addr = var.ip_version == 6 ? var.secondary_leader_host.ipv6 : var.secondary_leader_host.private_ip
primary_replication_status = jsondecode(enos_remote_exec.verify_replication_status_on_primary.stdout)
secondary_replication_status = jsondecode(enos_remote_exec.verify_replication_status_on_secondary.stdout)
}
resource "enos_remote_exec" "verify_replication_status_on_primary" {
environment = {
IP_VERSION = var.ip_version
PRIMARY_LEADER_ADDR = local.primary_leader_addr
SECONDARY_LEADER_ADDR = local.secondary_leader_addr
VAULT_ADDR = var.vault_addr
VAULT_INSTALL_DIR = var.vault_install_dir
}
scripts = [abspath("${path.module}/scripts/verify-replication-status.sh")]
transport = {
ssh = {
host = var.primary_leader_host.public_ip
}
}
}
resource "enos_remote_exec" "verify_replication_status_on_secondary" {
environment = {
IP_VERSION = var.ip_version
PRIMARY_LEADER_ADDR = local.primary_leader_addr
SECONDARY_LEADER_ADDR = local.secondary_leader_addr
VAULT_ADDR = var.vault_addr
VAULT_INSTALL_DIR = var.vault_install_dir
}
scripts = [abspath("${path.module}/scripts/verify-replication-status.sh")]
transport = {
ssh = {
host = var.secondary_leader_host.public_ip
}
}
}
output "primary_replication_status" {
value = local.primary_replication_status
}
output "known_primary_cluster_addrs" {
value = local.secondary_replication_status.data.known_primary_cluster_addrs
}
output "secondary_replication_status" {
value = local.secondary_replication_status
}
output "primary_replication_data_secondaries" {
value = local.primary_replication_status.data.secondaries
}
output "secondary_replication_data_primaries" {
value = local.secondary_replication_status.data.primaries
}

89
enos/modules/vault_verify_dr_replication/scripts/verify-replication-status.sh Normal file
View File

@ -0,0 +1,89 @@
#!/usr/bin/env bash
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: BUSL-1.1
# This script waits for the replication status to be established
# then verifies the dr replication between primary and
# secondary clusters
set -e
fail() {
echo "$1" 1>&2
exit 1
}
[[ -z "$IP_VERSION" ]] && fail "IP_VERSION env variable has not been set"
[[ -z "$PRIMARY_LEADER_ADDR" ]] && fail "PRIMARY_LEADER_ADDR env variable has not been set"
[[ -z "$SECONDARY_LEADER_ADDR" ]] && fail "SECONDARY_LEADER_ADDR env variable has not been set"
[[ -z "$VAULT_ADDR" ]] && fail "VAULT_ADDR env variable has not been set"
[[ -z "$VAULT_INSTALL_DIR" ]] && fail "VAULT_INSTALL_DIR env variable has not been set"
binpath=${VAULT_INSTALL_DIR}/vault
test -x "$binpath" || fail "unable to locate vault binary at $binpath"
retry() {
local retries=$1
shift
local count=0
until "$@"; do
wait=$((2 ** count))
count=$((count + 1))
if [ "$count" -lt "$retries" ]; then
sleep "$wait"
else
fail "$($binpath read -format=json sys/replication/dr/status)"
fi
done
}
check_dr_status() {
dr_status=$($binpath read -format=json sys/replication/dr/status)
cluster_state=$(jq -r '.data.state' <<< "$dr_status")
connection_mode=$(jq -r '.data.mode' <<< "$dr_status")
if [[ "$cluster_state" == 'idle' ]]; then
echo "replication cluster state is idle" 1>&2
return 1
fi
if [[ "$connection_mode" == "primary" ]]; then
connection_status=$(jq -r '.data.secondaries[0].connection_status' <<< "$dr_status")
if [[ "$connection_status" == 'disconnected' ]]; then
echo ".data.secondaries[0].connection_status from primary node is 'disconnected'" 1>&2
return 1
fi
# Confirm we are in a "running" state for the primary
if [[ "$cluster_state" != "running" ]]; then
echo "replication cluster primary state is not running" 1>&2
return 1
fi
else
connection_status=$(jq -r '.data.primaries[0].connection_status' <<< "$dr_status")
if [[ "$connection_status" == 'disconnected' ]]; then
echo ".data.primaries[0].connection_status from secondary node is 'disconnected'" 1>&2
return 1
fi
# Confirm we are in a "stream-wals" state for the secondary
if [[ "$cluster_state" != "stream-wals" ]]; then
echo "replication cluster primary state is not stream-wals" 1>&2
return 1
fi
known_primary_cluster_addrs=$(jq -r '.data.known_primary_cluster_addrs' <<< "$dr_status")
if ! echo "$known_primary_cluster_addrs" | grep -q "$PRIMARY_LEADER_ADDR"; then
echo "$PRIMARY_LEADER_ADDR is not in .data.known_primary_cluster_addrs: $known_primary_cluster_addrs" 1>&2
return 1
fi
fi
echo "$dr_status"
return 0
}
if [ "$IP_VERSION" != 4 ] && [ "$IP_VERSION" != 6 ]; then
fail "unsupported IP_VERSION: $IP_VERSION"
fi
# Retry for a while because it can take some time for replication to sync
retry 10 check_dr_status