From cdf3da406682e9de3b023c1fd3aaf3fb5acb5a67 Mon Sep 17 00:00:00 2001 From: "Luis (LT) Carbonell" Date: Thu, 5 Sep 2024 17:33:53 -0400 Subject: [PATCH] Add DR failover scenario to Enos (#28256) * Add DR failover scenario to Enos * Update enos/enos-scenario-dr-replication.hcl Co-authored-by: Ryan Cragun * Update enos/enos-qualities.hcl Co-authored-by: Ryan Cragun * Update enos/enos-scenario-dr-replication.hcl Co-authored-by: Ryan Cragun * Update enos/enos-scenario-dr-replication.hcl Co-authored-by: Ryan Cragun * Update enos/enos-qualities.hcl Co-authored-by: Ryan Cragun * Update enos/enos-scenario-dr-replication.hcl Co-authored-by: Ryan Cragun * Update enos/enos-scenario-dr-replication.hcl Co-authored-by: Ryan Cragun * Update enos/enos-scenario-dr-replication.hcl Co-authored-by: Ryan Cragun * Update enos/enos-scenario-pr-replication.hcl Co-authored-by: Ryan Cragun * remove superuser * Update enos/enos-scenario-dr-replication.hcl Co-authored-by: Ryan Cragun * Update enos/enos-scenario-dr-replication.hcl Co-authored-by: Ryan Cragun * Update enos/enos-scenario-dr-replication.hcl Co-authored-by: Ryan Cragun * Update enos/enos-scenario-dr-replication.hcl Co-authored-by: Ryan Cragun * Update enos/enos-scenario-dr-replication.hcl Co-authored-by: Ryan Cragun --------- Co-authored-by: Ryan Cragun --- enos/enos-dev-scenario-pr-replication.hcl | 34 +- enos/enos-modules.hcl | 97 +- enos/enos-qualities.hcl | 53 + enos/enos-scenario-dr-replication.hcl | 1258 +++++++++++++++++ ...n.hcl => enos-scenario-pr-replication.hcl} | 39 +- .../generate_dr_operation_token/main.tf | 82 ++ .../scripts/configure-vault-dr-primary.sh | 50 + .../generate_failover_secondary_token/main.tf | 98 ++ .../generate-failover-secondary-token.sh | 33 + .../generate_secondary_public_key/main.tf | 77 + enos/modules/generate_secondary_token/main.tf | 43 +- .../vault_failover_demote_dr_primary/main.tf | 63 + .../main.tf | 69 + .../vault_failover_update_dr_primary/main.tf | 76 + enos/modules/vault_setup_dr_primary/main.tf | 59 + .../scripts/configure-vault-dr-primary.sh | 17 + enos/modules/vault_setup_perf_primary/main.tf | 22 +- .../scripts/configure-vault-pr-primary.sh | 12 - .../main.tf | 32 +- .../vault_verify_dr_replication/main.tf | 117 ++ .../scripts/verify-replication-status.sh | 89 ++ 21 files changed, 2336 insertions(+), 84 deletions(-) create mode 100644 enos/enos-scenario-dr-replication.hcl rename enos/{enos-scenario-replication.hcl => enos-scenario-pr-replication.hcl} (97%) create mode 100644 enos/modules/generate_dr_operation_token/main.tf create mode 100755 enos/modules/generate_dr_operation_token/scripts/configure-vault-dr-primary.sh create mode 100644 enos/modules/generate_failover_secondary_token/main.tf create mode 100644 enos/modules/generate_failover_secondary_token/scripts/generate-failover-secondary-token.sh create mode 100644 enos/modules/generate_secondary_public_key/main.tf create mode 100644 enos/modules/vault_failover_demote_dr_primary/main.tf create mode 100644 enos/modules/vault_failover_promote_dr_secondary/main.tf create mode 100644 enos/modules/vault_failover_update_dr_primary/main.tf create mode 100644 enos/modules/vault_setup_dr_primary/main.tf create mode 100644 enos/modules/vault_setup_dr_primary/scripts/configure-vault-dr-primary.sh rename enos/modules/{vault_setup_perf_secondary => vault_setup_replication_secondary}/main.tf (53%) create mode 100644 enos/modules/vault_verify_dr_replication/main.tf create mode 100644 enos/modules/vault_verify_dr_replication/scripts/verify-replication-status.sh diff --git a/enos/enos-dev-scenario-pr-replication.hcl b/enos/enos-dev-scenario-pr-replication.hcl index 3a81d175d3..0a4f0494a1 100644 --- a/enos/enos-dev-scenario-pr-replication.hcl +++ b/enos/enos-dev-scenario-pr-replication.hcl @@ -756,10 +756,12 @@ scenario "dev_pr_replication" { } variables { - primary_leader_public_ip = step.get_primary_cluster_ips.leader_public_ip - vault_addr = step.create_primary_cluster.api_addr_localhost - vault_install_dir = local.vault_install_dir - vault_root_token = step.create_primary_cluster.root_token + ip_version = local.ip_version + primary_leader_host = step.get_primary_cluster_ips.leader_host + replication_type = "performance" + vault_addr = step.create_primary_cluster.api_addr_localhost + vault_install_dir = local.vault_install_dir + vault_root_token = step.create_primary_cluster.root_token } } @@ -776,10 +778,12 @@ scenario "dev_pr_replication" { } variables { - primary_leader_public_ip = step.get_primary_cluster_ips.leader_public_ip - vault_addr = step.create_primary_cluster.api_addr_localhost - vault_install_dir = local.vault_install_dir - vault_root_token = step.create_primary_cluster.root_token + ip_version = local.ip_version + primary_leader_host = step.get_primary_cluster_ips.leader_host + replication_type = "performance" + vault_addr = step.create_primary_cluster.api_addr_localhost + vault_install_dir = local.vault_install_dir + vault_root_token = step.create_primary_cluster.root_token } } @@ -787,7 +791,7 @@ scenario "dev_pr_replication" { description = <<-EOF Enable performance replication on the secondary using the new shared token. EOF - module = module.vault_setup_perf_secondary + module = module.vault_setup_replication_secondary depends_on = [step.generate_secondary_token] providers = { @@ -795,11 +799,13 @@ scenario "dev_pr_replication" { } variables { - secondary_leader_public_ip = step.get_secondary_cluster_ips.leader_public_ip - vault_addr = step.create_secondary_cluster.api_addr_localhost - vault_install_dir = local.vault_install_dir - vault_root_token = step.create_secondary_cluster.root_token - wrapping_token = step.generate_secondary_token.secondary_token + ip_version = local.ip_version + secondary_leader_host = step.get_secondary_cluster_ips.leader_host + replication_type = "performance" + vault_addr = step.create_secondary_cluster.api_addr_localhost + vault_install_dir = local.vault_install_dir + vault_root_token = step.create_secondary_cluster.root_token + wrapping_token = step.generate_secondary_token.secondary_token } } diff --git a/enos/enos-modules.hcl b/enos/enos-modules.hcl index a503cc783b..137e124596 100644 --- a/enos/enos-modules.hcl +++ b/enos/enos-modules.hcl @@ -56,6 +56,24 @@ module "get_local_metadata" { source = "./modules/get_local_metadata" } +module "generate_dr_operation_token" { + source = "./modules/generate_dr_operation_token" + + vault_install_dir = var.vault_install_dir +} + +module "generate_failover_secondary_token" { + source = "./modules/generate_failover_secondary_token" + + vault_install_dir = var.vault_install_dir +} + +module "generate_secondary_public_key" { + source = "./modules/generate_secondary_public_key" + + vault_install_dir = var.vault_install_dir +} + module "generate_secondary_token" { source = "./modules/generate_secondary_token" @@ -185,13 +203,43 @@ module "vault_get_cluster_ips" { vault_install_dir = var.vault_install_dir } +module "vault_failover_demote_dr_primary" { + source = "./modules/vault_failover_demote_dr_primary" + + vault_install_dir = var.vault_install_dir +} + +module "vault_failover_promote_dr_secondary" { + source = "./modules/vault_failover_promote_dr_secondary" + + vault_install_dir = var.vault_install_dir +} + +module "vault_failover_update_dr_primary" { + source = "./modules/vault_failover_update_dr_primary" + + vault_install_dir = var.vault_install_dir +} + module "vault_raft_remove_peer" { source = "./modules/vault_raft_remove_peer" vault_install_dir = var.vault_install_dir } -module "vault_setup_perf_secondary" { - source = "./modules/vault_setup_perf_secondary" +module "vault_setup_dr_primary" { + source = "./modules/vault_setup_dr_primary" + + vault_install_dir = var.vault_install_dir +} + +module "vault_setup_perf_primary" { + source = "./modules/vault_setup_perf_primary" + + vault_install_dir = var.vault_install_dir +} + +module "vault_setup_replication_secondary" { + source = "./modules/vault_setup_replication_secondary" vault_install_dir = var.vault_install_dir } @@ -227,6 +275,12 @@ module "vault_verify_autopilot" { vault_install_dir = var.vault_install_dir } +module "vault_verify_dr_replication" { + source = "./modules/vault_verify_dr_replication" + + vault_install_dir = var.vault_install_dir +} + module "vault_verify_raft_auto_join_voter" { source = "./modules/vault_verify_raft_auto_join_voter" @@ -234,11 +288,6 @@ module "vault_verify_raft_auto_join_voter" { vault_cluster_addr_port = global.ports["vault_cluster"]["port"] } -module "vault_verify_undo_logs" { - source = "./modules/vault_verify_undo_logs" - - vault_install_dir = var.vault_install_dir -} module "vault_verify_default_lcq" { source = "./modules/vault_verify_default_lcq" @@ -250,22 +299,6 @@ module "vault_verify_replication" { source = "./modules/vault_verify_replication" } -module "vault_verify_ui" { - source = "./modules/vault_verify_ui" -} - -module "vault_verify_unsealed" { - source = "./modules/vault_verify_unsealed" - - vault_install_dir = var.vault_install_dir -} - -module "vault_setup_perf_primary" { - source = "./modules/vault_setup_perf_primary" - - vault_install_dir = var.vault_install_dir -} - module "vault_verify_read_data" { source = "./modules/vault_verify_read_data" @@ -290,6 +323,22 @@ module "vault_verify_write_data" { vault_install_dir = var.vault_install_dir } +module "vault_verify_ui" { + source = "./modules/vault_verify_ui" +} + +module "vault_verify_undo_logs" { + source = "./modules/vault_verify_undo_logs" + + vault_install_dir = var.vault_install_dir +} + +module "vault_verify_unsealed" { + source = "./modules/vault_verify_unsealed" + + vault_install_dir = var.vault_install_dir +} + module "vault_wait_for_leader" { source = "./modules/vault_wait_for_leader" @@ -314,4 +363,4 @@ module "vault_verify_billing_start_date" { vault_install_dir = var.vault_install_dir vault_instance_count = var.vault_instance_count vault_cluster_addr_port = global.ports["vault_cluster"]["port"] -} \ No newline at end of file +} diff --git a/enos/enos-qualities.hcl b/enos/enos-qualities.hcl index 7a19759ac8..529990f6fc 100644 --- a/enos/enos-qualities.hcl +++ b/enos/enos-qualities.hcl @@ -120,6 +120,59 @@ quality "vault_api_sys_quotas_lease_count_read_max_leases_default" { EOF } +quality "vault_api_sys_replication_dr_primary_enable_write" { + description = <<-EOF + The v1/sys/replication/dr/primary/enable Vault API enables DR replication + EOF +} + +quality "vault_api_sys_replication_dr_primary_secondary_token_write" { + description = <<-EOF + The v1/sys/replication/dr/primary/secondary-token Vault API configures the DR replication + secondary token + EOF +} + +quality "vault_api_sys_replication_dr_secondary_enable_write" { + description = <<-EOF + The v1/sys/replication/dr/secondary/enable Vault API enables DR replication + EOF +} + +quality "vault_api_sys_replication_dr_read_connection_status_connected" { + description = <<-EOF + The v1/sys/replication/dr/status Vault API returns status info and the + 'connection_status' is correct for the given node + EOF +} + +quality "vault_api_sys_replication_dr_status_known_primary_cluster_addrs" { + description = <<-EOF + The v1/sys/replication/dr/status Vault API returns the DR replication status and + 'known_primary_cluster_address' is the expected primary cluster leader + EOF +} + +quality "vault_api_sys_replication_dr_status_read" { + description = <<-EOF + The v1/sys/replication/dr/status Vault API returns the DR replication status + EOF +} + +quality "vault_api_sys_replication_dr_status_read_cluster_address" { + description = <<-EOF + The v1/sys/replication/dr/status Vault API returns the DR replication status + and the '{primaries,secondaries}[*].cluster_address' is correct for the given node + EOF +} + +quality "vault_api_sys_replication_dr_status_read_state_not_idle" { + description = <<-EOF + The v1/sys/replication/dr/status Vault API returns the DR replication status + and the state is not idle + EOF +} + quality "vault_api_sys_replication_performance_primary_enable_write" { description = <<-EOF The v1/sys/replication/performance/primary/enable Vault API enables performance replication diff --git a/enos/enos-scenario-dr-replication.hcl b/enos/enos-scenario-dr-replication.hcl new file mode 100644 index 0000000000..452bdc3165 --- /dev/null +++ b/enos/enos-scenario-dr-replication.hcl @@ -0,0 +1,1258 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +scenario "dr_replication" { + description = <<-EOF + The DR replication scenario configures disaster recovery replication between two Vault clusters and + verifies behavior and failure tolerance. The build can be a local branch, any CRT built Vault + Enterprise artifact saved to the local machine, or any CRT built Vault Enterprise artifact in + the stable channel in Artifactory. + + The scenario deploys two Vault Enterprise clusters and establishes disaster recovery replication + between the primary cluster and the disaster recovery replication secondary cluster. Next, we write + test data to the primary cluster and verify that the data is replicated to the secondary cluster. + We then promote the secondary cluster to be the primary cluster and demote the primary cluster to be + the secondary cluster. We then update the secondary cluster to connect to the new primary cluster. + Finally, we verify that the secondary cluster is unsealed after enabling replication and verify the + disaster recovery replication status between the primary and secondary clusters. + + If you want to use the 'distro:leap' variant you must first accept SUSE's terms for the AWS + account. To verify that your account has agreed, sign-in to your AWS through Doormat, + and visit the following links to verify your subscription or subscribe: + arm64 AMI: https://aws.amazon.com/marketplace/server/procurement?productId=a516e959-df54-4035-bb1a-63599b7a6df9 + amd64 AMI: https://aws.amazon.com/marketplace/server/procurement?productId=5535c495-72d4-4355-b169-54ffa874f849 + EOF + + matrix { + arch = global.archs + artifact_source = global.artifact_sources + artifact_type = global.artifact_types + config_mode = global.config_modes + consul_edition = global.consul_editions + consul_version = global.consul_versions + distro = global.distros + edition = global.enterprise_editions + ip_version = global.ip_versions + primary_backend = global.backends + primary_seal = global.seals + secondary_backend = global.backends + secondary_seal = global.seals + + // Our local builder always creates bundles + exclude { + artifact_source = ["local"] + artifact_type = ["package"] + } + + // PKCS#11 can only be used on ent.hsm and ent.hsm.fips1402. + exclude { + primary_seal = ["pkcs11"] + edition = [for e in matrix.edition : e if !strcontains(e, "hsm")] + } + + exclude { + secondary_seal = ["pkcs11"] + edition = [for e in matrix.edition : e if !strcontains(e, "hsm")] + } + + // softhsm packages not available for leap/sles. + exclude { + primary_seal = ["pkcs11"] + distro = ["leap", "sles"] + } + + exclude { + secondary_seal = ["pkcs11"] + distro = ["leap", "sles"] + } + + // Testing in IPV6 mode is currently implemented for integrated Raft storage only + exclude { + ip_version = ["6"] + primary_backend = ["consul"] + } + + exclude { + ip_version = ["6"] + secondary_backend = ["consul"] + } + } + + terraform_cli = terraform_cli.default + terraform = terraform.default + providers = [ + provider.aws.default, + provider.enos.ec2_user, + provider.enos.ubuntu + ] + + locals { + artifact_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_artifact_path) : null + enos_provider = { + amzn = provider.enos.ec2_user + leap = provider.enos.ec2_user + rhel = provider.enos.ec2_user + sles = provider.enos.ec2_user + ubuntu = provider.enos.ubuntu + } + manage_service = matrix.artifact_type == "bundle" + vault_install_dir = matrix.artifact_type == "bundle" ? var.vault_install_dir : global.vault_install_dir[matrix.artifact_type] + } + + step "build_vault" { + description = global.description.build_vault + module = "build_${matrix.artifact_source}" + + variables { + build_tags = var.vault_local_build_tags != null ? var.vault_local_build_tags : global.build_tags[matrix.edition] + artifact_path = local.artifact_path + goarch = matrix.arch + goos = "linux" + artifactory_host = matrix.artifact_source == "artifactory" ? var.artifactory_host : null + artifactory_repo = matrix.artifact_source == "artifactory" ? var.artifactory_repo : null + artifactory_username = matrix.artifact_source == "artifactory" ? var.artifactory_username : null + artifactory_token = matrix.artifact_source == "artifactory" ? var.artifactory_token : null + arch = matrix.artifact_source == "artifactory" ? matrix.arch : null + product_version = var.vault_product_version + artifact_type = matrix.artifact_type + distro = matrix.artifact_source == "artifactory" ? matrix.distro : null + edition = matrix.artifact_source == "artifactory" ? matrix.edition : null + revision = var.vault_revision + } + } + + step "ec2_info" { + description = global.description.ec2_info + module = module.ec2_info + } + + step "create_vpc" { + description = global.description.create_vpc + module = module.create_vpc + + variables { + common_tags = global.tags + ip_version = matrix.ip_version + } + } + + // This step reads the contents of the backend license if we're using a Consul backend and + // an "ent" Consul edition. + step "read_backend_license" { + description = global.description.read_backend_license + skip_step = (matrix.primary_backend == "raft" && matrix.secondary_backend == "raft") || matrix.consul_edition == "ce" + module = module.read_license + + variables { + file_name = global.backend_license_path + } + } + + step "read_vault_license" { + description = global.description.read_vault_license + module = module.read_license + + variables { + file_name = abspath(joinpath(path.root, "./support/vault.hclic")) + } + } + + step "create_primary_seal_key" { + description = global.description.create_seal_key + module = "seal_${matrix.primary_seal}" + depends_on = [step.create_vpc] + + providers = { + enos = provider.enos.ubuntu + } + + variables { + cluster_id = step.create_vpc.id + cluster_meta = "primary" + common_tags = global.tags + } + } + + step "create_secondary_seal_key" { + description = global.description.create_seal_key + module = "seal_${matrix.secondary_seal}" + depends_on = [step.create_vpc] + + providers = { + enos = provider.enos.ubuntu + } + + variables { + cluster_id = step.create_vpc.id + cluster_meta = "secondary" + common_tags = global.tags + other_resources = step.create_primary_seal_key.resource_names + } + } + + // Create all of our instances for both primary and secondary clusters + step "create_primary_cluster_targets" { + description = global.description.create_vault_cluster_targets + module = module.target_ec2_instances + depends_on = [ + step.create_vpc, + ] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]] + cluster_tag_key = global.vault_tag_key + common_tags = global.tags + seal_key_names = step.create_primary_seal_key.resource_names + vpc_id = step.create_vpc.id + } + } + + step "create_primary_cluster_backend_targets" { + description = global.description.create_vault_cluster_targets + module = matrix.primary_backend == "consul" ? module.target_ec2_instances : module.target_ec2_shim + depends_on = [ + step.create_vpc, + ] + + providers = { + enos = provider.enos.ubuntu + } + + variables { + ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"][global.distro_version["ubuntu"]] + cluster_tag_key = global.backend_tag_key + common_tags = global.tags + seal_key_names = step.create_primary_seal_key.resource_names + vpc_id = step.create_vpc.id + } + } + + step "create_secondary_cluster_targets" { + description = global.description.create_vault_cluster_targets + module = module.target_ec2_instances + depends_on = [step.create_vpc] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]] + cluster_tag_key = global.vault_tag_key + common_tags = global.tags + seal_key_names = step.create_secondary_seal_key.resource_names + vpc_id = step.create_vpc.id + } + } + + step "create_secondary_cluster_backend_targets" { + description = global.description.create_vault_cluster_targets + module = matrix.secondary_backend == "consul" ? module.target_ec2_instances : module.target_ec2_shim + depends_on = [step.create_vpc] + + providers = { + enos = provider.enos.ubuntu + } + + variables { + ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"][global.distro_version["ubuntu"]] + cluster_tag_key = global.backend_tag_key + common_tags = global.tags + seal_key_names = step.create_secondary_seal_key.resource_names + vpc_id = step.create_vpc.id + } + } + + step "create_primary_backend_cluster" { + description = global.description.create_backend_cluster + module = "backend_${matrix.primary_backend}" + depends_on = [ + step.create_primary_cluster_backend_targets, + ] + + providers = { + enos = provider.enos.ubuntu + } + + verifies = [ + // verified in modules + quality.consul_autojoin_aws, + quality.consul_config_file, + quality.consul_ha_leader_election, + quality.consul_service_start_server, + // verified in enos_consul_start resource + quality.consul_api_agent_host_read, + quality.consul_api_health_node_read, + quality.consul_api_operator_raft_config_read, + quality.consul_cli_validate, + quality.consul_health_state_passing_read_nodes_minimum, + quality.consul_operator_raft_configuration_read_voters_minimum, + quality.consul_service_systemd_notified, + quality.consul_service_systemd_unit, + ] + + variables { + cluster_name = step.create_primary_cluster_backend_targets.cluster_name + cluster_tag_key = global.backend_tag_key + hosts = step.create_primary_cluster_backend_targets.hosts + license = (matrix.primary_backend == "consul" && matrix.consul_edition == "ent") ? step.read_backend_license.license : null + release = { + edition = matrix.consul_edition + version = matrix.consul_version + } + } + } + + step "create_primary_cluster" { + description = global.description.create_vault_cluster + module = module.vault_cluster + depends_on = [ + step.create_primary_backend_cluster, + step.build_vault, + step.create_primary_cluster_targets + ] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = [ + // verified in modules + quality.consul_service_start_client, + quality.vault_artifact_bundle, + quality.vault_artifact_deb, + quality.vault_artifact_rpm, + quality.vault_audit_log, + quality.vault_audit_socket, + quality.vault_audit_syslog, + quality.vault_autojoin_aws, + quality.vault_config_env_variables, + quality.vault_config_file, + quality.vault_config_log_level, + quality.vault_init, + quality.vault_license_required_ent, + quality.vault_listener_ipv4, + quality.vault_listener_ipv6, + quality.vault_service_start, + quality.vault_storage_backend_consul, + quality.vault_storage_backend_raft, + // verified in enos_vault_start resource + quality.vault_api_sys_config_read, + quality.vault_api_sys_ha_status_read, + quality.vault_api_sys_health_read, + quality.vault_api_sys_host_info_read, + quality.vault_api_sys_replication_status_read, + quality.vault_api_sys_seal_status_api_read_matches_sys_health, + quality.vault_api_sys_storage_raft_autopilot_configuration_read, + quality.vault_api_sys_storage_raft_autopilot_state_read, + quality.vault_api_sys_storage_raft_configuration_read, + quality.vault_cli_status_exit_code, + quality.vault_service_systemd_notified, + quality.vault_service_systemd_unit, + ] + + variables { + artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null + backend_cluster_name = step.create_primary_cluster_backend_targets.cluster_name + backend_cluster_tag_key = global.backend_tag_key + config_mode = matrix.config_mode + consul_license = (matrix.primary_backend == "consul" && matrix.consul_edition == "ent") ? step.read_backend_license.license : null + cluster_name = step.create_primary_cluster_targets.cluster_name + consul_release = matrix.primary_backend == "consul" ? { + edition = matrix.consul_edition + version = matrix.consul_version + } : null + enable_audit_devices = var.vault_enable_audit_devices + hosts = step.create_primary_cluster_targets.hosts + install_dir = global.vault_install_dir[matrix.artifact_type] + ip_version = matrix.ip_version + license = matrix.edition != "ce" ? step.read_vault_license.license : null + local_artifact_path = local.artifact_path + manage_service = local.manage_service + packages = concat(global.packages, global.distro_packages[matrix.distro][global.distro_version[matrix.distro]]) + seal_attributes = step.create_primary_seal_key.attributes + seal_type = matrix.primary_seal + storage_backend = matrix.primary_backend + } + } + + step "get_local_metadata" { + skip_step = matrix.artifact_source != "local" + module = module.get_local_metadata + } + + step "wait_for_primary_cluster_leader" { + description = global.description.wait_for_cluster_to_have_leader + module = module.vault_wait_for_leader + depends_on = [step.create_primary_cluster] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = [ + quality.vault_api_sys_leader_read, + quality.vault_unseal_ha_leader_election, + ] + + variables { + hosts = step.create_primary_cluster_targets.hosts + ip_version = matrix.ip_version + timeout = 120 // seconds + vault_addr = step.create_primary_cluster.api_addr_localhost + vault_install_dir = local.vault_install_dir + vault_root_token = step.create_primary_cluster.root_token + } + } + + step "create_secondary_backend_cluster" { + description = global.description.create_backend_cluster + module = "backend_${matrix.secondary_backend}" + depends_on = [ + step.create_secondary_cluster_backend_targets + ] + + providers = { + enos = provider.enos.ubuntu + } + + variables { + cluster_name = step.create_secondary_cluster_backend_targets.cluster_name + cluster_tag_key = global.backend_tag_key + hosts = step.create_secondary_cluster_backend_targets.hosts + license = (matrix.secondary_backend == "consul" && matrix.consul_edition == "ent") ? step.read_backend_license.license : null + release = { + edition = matrix.consul_edition + version = matrix.consul_version + } + } + } + + step "create_secondary_cluster" { + module = module.vault_cluster + depends_on = [ + step.create_secondary_backend_cluster, + step.build_vault, + step.create_secondary_cluster_targets + ] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = [ + // verified in modules + quality.consul_autojoin_aws, + quality.consul_config_file, + quality.consul_ha_leader_election, + quality.consul_service_start_client, + // verified in enos_consul_start resource + quality.consul_api_agent_host_read, + quality.consul_api_health_node_read, + quality.consul_api_operator_raft_config_read, + quality.consul_health_state_passing_read_nodes_minimum, + quality.consul_operator_raft_configuration_read_voters_minimum, + quality.consul_service_systemd_notified, + quality.consul_service_systemd_unit, + ] + + variables { + artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null + backend_cluster_name = step.create_secondary_cluster_backend_targets.cluster_name + backend_cluster_tag_key = global.backend_tag_key + config_mode = matrix.config_mode + consul_license = (matrix.secondary_backend == "consul" && matrix.consul_edition == "ent") ? step.read_backend_license.license : null + cluster_name = step.create_secondary_cluster_targets.cluster_name + consul_release = matrix.secondary_backend == "consul" ? { + edition = matrix.consul_edition + version = matrix.consul_version + } : null + enable_audit_devices = var.vault_enable_audit_devices + hosts = step.create_secondary_cluster_targets.hosts + install_dir = global.vault_install_dir[matrix.artifact_type] + ip_version = matrix.ip_version + license = matrix.edition != "ce" ? step.read_vault_license.license : null + local_artifact_path = local.artifact_path + manage_service = local.manage_service + packages = concat(global.packages, global.distro_packages[matrix.distro][global.distro_version[matrix.distro]]) + seal_attributes = step.create_secondary_seal_key.attributes + seal_type = matrix.secondary_seal + storage_backend = matrix.secondary_backend + } + } + + step "wait_for_secondary_cluster_leader" { + description = global.description.wait_for_cluster_to_have_leader + module = module.vault_wait_for_leader + depends_on = [step.create_secondary_cluster] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = [ + quality.vault_api_sys_leader_read, + quality.vault_unseal_ha_leader_election, + ] + + variables { + hosts = step.create_secondary_cluster_targets.hosts + ip_version = matrix.ip_version + timeout = 120 // seconds + vault_addr = step.create_secondary_cluster.api_addr_localhost + vault_install_dir = local.vault_install_dir + vault_root_token = step.create_secondary_cluster.root_token + } + } + + step "verify_that_vault_primary_cluster_is_unsealed" { + description = global.description.verify_vault_unsealed + module = module.vault_verify_unsealed + depends_on = [ + step.create_primary_cluster, + step.wait_for_primary_cluster_leader, + ] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = [ + quality.vault_auto_unseals_after_autopilot_upgrade, + quality.vault_seal_awskms, + quality.vault_seal_pkcs11, + quality.vault_seal_shamir, + ] + + variables { + hosts = step.create_primary_cluster_targets.hosts + vault_addr = step.create_primary_cluster.api_addr_localhost + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + } + } + + step "verify_that_vault_secondary_cluster_is_unsealed" { + description = global.description.verify_vault_unsealed + module = module.vault_verify_unsealed + depends_on = [ + step.create_secondary_cluster, + step.wait_for_secondary_cluster_leader, + ] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = [ + quality.vault_auto_unseals_after_autopilot_upgrade, + quality.vault_seal_awskms, + quality.vault_seal_pkcs11, + quality.vault_seal_shamir, + ] + + variables { + hosts = step.create_secondary_cluster_targets.hosts + vault_addr = step.create_secondary_cluster.api_addr_localhost + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + } + } + + step "get_primary_cluster_ips" { + description = global.description.get_vault_cluster_ip_addresses + module = module.vault_get_cluster_ips + depends_on = [step.verify_that_vault_primary_cluster_is_unsealed] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = [ + quality.vault_api_sys_ha_status_read, + quality.vault_api_sys_leader_read, + quality.vault_cli_operator_members, + ] + + variables { + hosts = step.create_primary_cluster_targets.hosts + ip_version = matrix.ip_version + vault_addr = step.create_primary_cluster.api_addr_localhost + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + vault_root_token = step.create_primary_cluster.root_token + } + } + + step "get_secondary_cluster_ips" { + description = global.description.get_vault_cluster_ip_addresses + module = module.vault_get_cluster_ips + depends_on = [step.verify_that_vault_secondary_cluster_is_unsealed] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = [ + quality.vault_api_sys_ha_status_read, + quality.vault_api_sys_leader_read, + quality.vault_cli_operator_members, + ] + + variables { + hosts = step.create_secondary_cluster_targets.hosts + ip_version = matrix.ip_version + vault_addr = step.create_secondary_cluster.api_addr_localhost + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + vault_root_token = step.create_secondary_cluster.root_token + } + } + + step "verify_vault_version" { + description = global.description.verify_vault_version + module = module.vault_verify_version + depends_on = [step.get_primary_cluster_ips] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = [ + quality.vault_api_sys_version_history_keys, + quality.vault_api_sys_version_history_key_info, + quality.vault_version_build_date, + quality.vault_version_edition, + quality.vault_version_release, + ] + + variables { + hosts = step.create_primary_cluster_targets.hosts + vault_addr = step.create_primary_cluster.api_addr_localhost + vault_edition = matrix.edition + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version + vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision + vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date + vault_root_token = step.create_primary_cluster.root_token + } + } + + step "verify_ui" { + description = global.description.verify_ui + module = module.vault_verify_ui + depends_on = [step.get_primary_cluster_ips] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = quality.vault_ui_assets + + variables { + vault_addr = step.create_primary_cluster.api_addr_localhost + hosts = step.create_primary_cluster_targets.hosts + } + } + + step "write_test_data_on_primary" { + description = global.description.verify_write_test_data + module = module.vault_verify_write_data + depends_on = [step.get_primary_cluster_ips] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = [ + quality.vault_mount_auth, + quality.vault_mount_kv, + quality.vault_secrets_auth_user_policy_write, + quality.vault_secrets_kv_write, + ] + + variables { + hosts = step.create_primary_cluster_targets.hosts + leader_host = step.get_primary_cluster_ips.leader_host + vault_addr = step.create_primary_cluster.api_addr_localhost + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + vault_root_token = step.create_primary_cluster.root_token + } + } + + # ================================================ + # DISASTER RECOVERY (DR) REPLICATION SETUP + # ================================================ + # 1. Configure DR primary replication on cluster A. + # 2. Generate secondary token on cluster A. + # 3. Configure DR secondary replication on cluster B. + # 4. Confirm replication status on both clusters. + + + step "configure_dr_replication_primary" { + description = <<-EOF + Create the necessary superuser auth policy necessary for DR replication, assign it + to a our previously create test user, and enable DR replication on the primary + cluster. + EOF + module = module.vault_setup_dr_primary + depends_on = [ + step.get_primary_cluster_ips, + step.get_secondary_cluster_ips, + step.write_test_data_on_primary + ] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = [ + quality.vault_api_sys_auth_userpass_user_write, + quality.vault_api_sys_policy_write, + quality.vault_api_sys_replication_dr_primary_enable_write, + quality.vault_cli_policy_write, + ] + + variables { + ip_version = matrix.ip_version + primary_leader_host = step.get_primary_cluster_ips.leader_host + vault_addr = step.create_primary_cluster.api_addr_localhost + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + vault_root_token = step.create_primary_cluster.root_token + } + } + + step "generate_secondary_token" { + description = <<-EOF + Generate a random token and configure the DR replication primary secondary-token and + configure the Vault cluster primary replication with the token. Export the wrapping token + so that secondary clusters can utilize it. + EOF + module = module.generate_secondary_token + depends_on = [step.configure_dr_replication_primary] + + verifies = quality.vault_api_sys_replication_dr_primary_secondary_token_write + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + ip_version = matrix.ip_version + primary_leader_host = step.get_primary_cluster_ips.leader_host + replication_type = "dr" + vault_addr = step.create_primary_cluster.api_addr_localhost + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + vault_root_token = step.create_primary_cluster.root_token + } + } + + step "configure_dr_replication_secondary" { + description = <<-EOF + Enable dr replication on the secondary cluster with the wrapping token created by + the primary cluster. + EOF + module = module.vault_setup_replication_secondary + depends_on = [step.generate_secondary_token] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = quality.vault_api_sys_replication_dr_secondary_enable_write + + variables { + ip_version = matrix.ip_version + secondary_leader_host = step.get_secondary_cluster_ips.leader_host + replication_type = "dr" + vault_addr = step.create_secondary_cluster.api_addr_localhost + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + vault_root_token = step.create_secondary_cluster.root_token + wrapping_token = step.generate_secondary_token.secondary_token + } + } + + step "unseal_secondary_followers" { + description = <<-EOF + After replication is enabled the secondary cluster followers need to be unsealed. + Secondary unseal keys are passed differently depending primary and secondary seal + type combinations. See the guide for more information: + https://developer.hashicorp.com/vault/docs/enterprise/replication#seals + EOF + module = module.vault_unseal_nodes + depends_on = [ + step.create_primary_cluster, + step.create_secondary_cluster, + step.get_secondary_cluster_ips, + step.configure_dr_replication_secondary + ] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + hosts = step.get_secondary_cluster_ips.follower_hosts + vault_addr = step.create_secondary_cluster.api_addr_localhost + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + vault_seal_type = matrix.primary_seal == "shamir" ? matrix.primary_seal : matrix.secondary_seal + vault_unseal_keys = matrix.primary_seal == "shamir" ? step.create_primary_cluster.unseal_keys_hex : step.create_primary_cluster.recovery_keys_hex + } + } + + step "verify_secondary_cluster_is_unsealed_after_enabling_replication" { + description = global.description.verify_vault_unsealed + module = module.vault_verify_unsealed + depends_on = [ + step.unseal_secondary_followers + ] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = [ + quality.vault_auto_unseals_after_autopilot_upgrade, + quality.vault_seal_awskms, + quality.vault_seal_pkcs11, + quality.vault_seal_shamir, + ] + + variables { + hosts = step.create_secondary_cluster_targets.hosts + vault_addr = step.create_secondary_cluster.api_addr_localhost + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + } + } + + step "verify_dr_replication" { + description = <<-EOF + Verify that the DR replication status meets our expectations after enabling replication + and ensuring that all secondary nodes are unsealed. + EOF + module = module.vault_verify_dr_replication + depends_on = [step.configure_dr_replication_secondary] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = [ + quality.vault_api_sys_replication_dr_read_connection_status_connected, + quality.vault_api_sys_replication_dr_status_read, + quality.vault_api_sys_replication_dr_status_read_cluster_address, + quality.vault_api_sys_replication_dr_status_read_state_not_idle, + quality.vault_api_sys_replication_dr_status_known_primary_cluster_addrs, + ] + + variables { + ip_version = matrix.ip_version + primary_leader_host = step.get_primary_cluster_ips.leader_host + secondary_leader_host = step.get_secondary_cluster_ips.leader_host + vault_addr = step.create_primary_cluster.api_addr_localhost + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + } + } + + # ============================== + # FAILOVER SCENARIO STEPS + # ============================== + # 1. Generate a batch DR operation token. + # 2. Promote the current secondary cluster B to become the new primary cluster. + # 3. Demote cluster A to secondary status. + # 4. Test access to Vault data on the new primary cluster B. + # 5. Point demoted cluster A to the new primary cluster B (Multistep process). + # 6. Verify that the data is replicated to the new primary cluster B. + + step "generate_batch_dr_operation_token" { + description = <<-EOF + Generate a batch DR operation token that you can use to promote and demote clusters as needed. + EOF + module = module.generate_dr_operation_token + depends_on = [step.verify_dr_replication] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + vault_install_dir = local.vault_install_dir + ip_version = matrix.ip_version + primary_leader_host = step.get_primary_cluster_ips.leader_host + vault_addr = step.create_primary_cluster.api_addr_localhost + vault_root_token = step.create_primary_cluster.root_token + storage_backend = matrix.primary_backend + } + } + + step "vault_failover_promote_dr_secondary_cluster" { + description = <<-EOF + Promote the secondary cluster to be the primary cluster. This step will also + generate a new DR operation token for the secondary cluster to connect to the new + primary cluster. + EOF + module = module.vault_failover_promote_dr_secondary + depends_on = [step.generate_batch_dr_operation_token] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + vault_install_dir = local.vault_install_dir + vault_addr = step.create_secondary_cluster.api_addr_localhost + ip_version = matrix.ip_version + secondary_leader_host = step.get_secondary_cluster_ips.leader_host + vault_root_token = step.create_secondary_cluster.root_token + dr_operation_token = step.generate_batch_dr_operation_token.dr_operation_token + } + } + + step "wait_for_promoted_cluster_leader" { + description = global.description.wait_for_cluster_to_have_leader + module = module.vault_wait_for_leader + depends_on = [step.vault_failover_promote_dr_secondary_cluster] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = [ + quality.vault_api_sys_leader_read, + quality.vault_unseal_ha_leader_election, + ] + + variables { + hosts = step.create_secondary_cluster_targets.hosts + ip_version = matrix.ip_version + timeout = 120 // seconds + vault_addr = step.create_secondary_cluster.api_addr_localhost + vault_install_dir = local.vault_install_dir + vault_root_token = step.create_secondary_cluster.root_token + } + } + + step "vault_failover_demote_dr_primary_cluster" { + description = <<-EOF + Demote the primary cluster to be the secondary cluster. This step will also + generate a new DR operation token for the secondary cluster to connect to the new + primary cluster. + EOF + module = module.vault_failover_demote_dr_primary + depends_on = [step.wait_for_promoted_cluster_leader] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + ip_version = matrix.ip_version + primary_leader_host = step.get_primary_cluster_ips.leader_host + vault_addr = step.create_primary_cluster.api_addr_localhost + vault_install_dir = local.vault_install_dir + vault_root_token = step.create_primary_cluster.root_token + } + } + + step "wait_for_demoted_cluster_leader" { + description = global.description.wait_for_cluster_to_have_leader + module = module.vault_wait_for_leader + depends_on = [step.vault_failover_demote_dr_primary_cluster] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = [ + quality.vault_api_sys_leader_read, + quality.vault_unseal_ha_leader_election, + ] + + variables { + hosts = step.create_primary_cluster_targets.hosts + ip_version = matrix.ip_version + timeout = 120 // seconds + vault_addr = step.create_primary_cluster.api_addr_localhost + vault_install_dir = local.vault_install_dir + vault_root_token = step.create_primary_cluster.root_token + } + } + + step "verify_replicated_data_during_failover" { + description = global.description.verify_read_test_data + module = module.vault_verify_read_data + depends_on = [ + step.wait_for_demoted_cluster_leader + ] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = quality.vault_secrets_kv_read + + variables { + hosts = step.get_secondary_cluster_ips.follower_hosts + vault_addr = step.create_secondary_cluster.api_addr_localhost + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + } + } + + step "generate_demoted_secondary_public_key" { + description = <<-EOF + Generate a random token and configure the DR replication primary secondary-token and + configure the Vault cluster primary replication with the token. Export the wrapping token + so that secondary clusters can utilize it. + EOF + module = module.generate_secondary_public_key + depends_on = [step.verify_replicated_data_during_failover] + + verifies = quality.vault_api_sys_replication_dr_primary_secondary_token_write + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + ip_version = matrix.ip_version + primary_leader_host = step.get_primary_cluster_ips.leader_host + vault_addr = step.create_primary_cluster.api_addr_localhost + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + vault_root_token = step.create_primary_cluster.root_token + } + } + + step "generate_demoted_secondary_token" { + description = <<-EOF + Generate a random token and configure the DR replication primary secondary-token and + configure the Vault cluster primary replication with the token. Export the wrapping token + so that secondary clusters can utilize it. + EOF + module = module.generate_failover_secondary_token + depends_on = [step.generate_demoted_secondary_public_key] + + verifies = quality.vault_api_sys_replication_dr_primary_secondary_token_write + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + ip_version = matrix.ip_version + primary_leader_host = step.get_secondary_cluster_ips.leader_host + secondary_public_key = step.generate_demoted_secondary_public_key.secondary_public_key + vault_addr = step.create_secondary_cluster.api_addr_localhost + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + vault_root_token = step.create_primary_cluster.root_token + } + } + + step "vault_failover_update_dr_primary_cluster" { + description = <<-EOF + Update the secondary cluster to connect to the new primary cluster. + EOF + module = module.vault_failover_update_dr_primary + depends_on = [ + step.generate_demoted_secondary_token, + step.vault_failover_demote_dr_primary_cluster + ] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + vault_install_dir = local.vault_install_dir + ip_version = matrix.ip_version + secondary_leader_host = step.get_primary_cluster_ips.leader_host + vault_addr = step.create_primary_cluster.api_addr_localhost + vault_root_token = step.create_primary_cluster.root_token + dr_operation_token = step.generate_batch_dr_operation_token.dr_operation_token + wrapping_token = step.generate_demoted_secondary_token.secondary_token + } + } + + step "verify_failover_dr_replication" { + description = <<-EOF + Verify that the DR replication status meets our expectations after enabling replication + and ensuring that all secondary nodes are unsealed. + EOF + module = module.vault_verify_dr_replication + depends_on = [step.vault_failover_update_dr_primary_cluster] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = [ + quality.vault_api_sys_replication_dr_read_connection_status_connected, + quality.vault_api_sys_replication_dr_status_read, + quality.vault_api_sys_replication_dr_status_read_cluster_address, + quality.vault_api_sys_replication_dr_status_read_state_not_idle, + quality.vault_api_sys_replication_dr_status_known_primary_cluster_addrs, + ] + + variables { + ip_version = matrix.ip_version + primary_leader_host = step.get_secondary_cluster_ips.leader_host + secondary_leader_host = step.get_primary_cluster_ips.leader_host + vault_addr = step.create_primary_cluster.api_addr_localhost + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + } + } + + step "verify_failover_replicated_data" { + description = global.description.verify_read_test_data + module = module.vault_verify_read_data + depends_on = [ + step.verify_dr_replication, + step.get_secondary_cluster_ips, + step.write_test_data_on_primary, + step.verify_failover_dr_replication + ] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = quality.vault_secrets_kv_read + + variables { + hosts = step.get_secondary_cluster_ips.follower_hosts + vault_addr = step.create_secondary_cluster.api_addr_localhost + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + } + } + + // Output the results of the scenario. + + output "audit_device_file_path" { + description = "The file path for the file audit device, if enabled" + value = step.create_primary_cluster.audit_device_file_path + } + + output "primary_cluster_hosts" { + description = "The Vault primary cluster target hosts" + value = step.create_primary_cluster_targets.hosts + } + + output "primary_cluster_root_token" { + description = "The Vault primary cluster root token" + value = step.create_primary_cluster.root_token + } + + output "primary_cluster_unseal_keys_b64" { + description = "The Vault primary cluster unseal keys" + value = step.create_primary_cluster.unseal_keys_b64 + } + + output "primary_cluster_unseal_keys_hex" { + description = "The Vault primary cluster unseal keys hex" + value = step.create_primary_cluster.unseal_keys_hex + } + + output "primary_cluster_recovery_key_shares" { + description = "The Vault primary cluster recovery key shares" + value = step.create_primary_cluster.recovery_key_shares + } + + output "primary_cluster_recovery_keys_b64" { + description = "The Vault primary cluster recovery keys b64" + value = step.create_primary_cluster.recovery_keys_b64 + } + + output "primary_cluster_recovery_keys_hex" { + description = "The Vault primary cluster recovery keys hex" + value = step.create_primary_cluster.recovery_keys_hex + } + + output "secondary_cluster_hosts" { + description = "The Vault secondary cluster public IPs" + value = step.create_secondary_cluster_targets.hosts + } + + output "secondary_cluster_root_token" { + description = "The Vault secondary cluster root token" + value = step.create_secondary_cluster.root_token + } + + output "dr_secondary_token" { + description = "The dr secondary replication token" + value = step.generate_secondary_token.secondary_token + } + + output "batch_dr_operation_token" { + description = "The dr primary replication token" + value = step.generate_batch_dr_operation_token.dr_operation_token + } + + output "demoted_secondary_public_key" { + description = "The dr secondary public key" + value = step.generate_demoted_secondary_public_key.secondary_public_key + } + + output "demoted_secondary_token" { + description = "The dr secondary public key" + value = step.generate_demoted_secondary_token.secondary_token + } + + output "initial_primary_replication_status" { + description = "The Vault primary cluster dr replication status" + value = step.verify_dr_replication.primary_replication_status + } + + output "initial_known_primary_cluster_addresses" { + description = "The Vault primary cluster known primary cluster addresses" + value = step.verify_dr_replication.known_primary_cluster_addrs + } + + output "initial_secondary_dr_replication_status" { + description = "The Vault secondary cluster dr replication status" + value = step.verify_dr_replication.secondary_replication_status + } + + output "intial_primary_replication_data_secondaries" { + description = "The Vault primary cluster secondaries connection status" + value = step.verify_dr_replication.primary_replication_data_secondaries + } + + output "initial_secondary_replication_data_primaries" { + description = "The Vault secondary cluster primaries connection status" + value = step.verify_dr_replication.secondary_replication_data_primaries + } + + output "get_primary_cluster_ips_leader" { + description = "The Vault updated primary cluster dr replication status" + value = step.get_primary_cluster_ips.leader_public_ip + } + + output "get_secondary_cluster_ips_leader" { + description = "The Vault updated primary cluster dr replication status" + value = step.get_secondary_cluster_ips.leader_public_ip + } + + output "failover_primary_replication_status" { + description = "The Vault updated primary cluster dr replication status" + value = step.verify_failover_dr_replication.primary_replication_status + } + + output "failover_known_primary_cluster_addresses" { + description = "The Vault secondary cluster dr replication status" + value = step.verify_failover_dr_replication.known_primary_cluster_addrs + } + + output "failover_secondary_replication_status" { + description = "The Vault updated secondary cluster dr replication status" + value = step.verify_failover_dr_replication.secondary_replication_status + } + + output "failover_primary_replication_data_secondaries" { + description = "The Vault updated primary cluster secondaries connection status" + value = step.verify_failover_dr_replication.primary_replication_data_secondaries + } + + output "failover_secondary_replication_data_primaries" { + description = "The Vault updated secondary cluster primaries connection status" + value = step.verify_failover_dr_replication.secondary_replication_data_primaries + } +} diff --git a/enos/enos-scenario-replication.hcl b/enos/enos-scenario-pr-replication.hcl similarity index 97% rename from enos/enos-scenario-replication.hcl rename to enos/enos-scenario-pr-replication.hcl index 9c38096d71..56456e1b68 100644 --- a/enos/enos-scenario-replication.hcl +++ b/enos/enos-scenario-pr-replication.hcl @@ -1,9 +1,9 @@ // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: BUSL-1.1 -scenario "replication" { +scenario "pr_replication" { description = <<-EOF - The replication scenario configures performance replication between two Vault clusters and + The PR replication scenario configures performance replication between two Vault clusters and verifies behavior and failure tolerance. The build can be a local branch, any CRT built Vault Enterprise artifact saved to the local machine, or any CRT built Vault Enterprise artifact in the stable channel in Artifactory. @@ -704,7 +704,7 @@ scenario "replication" { step "configure_performance_replication_primary" { description = <<-EOF - Create the necessary superuser auth policy necessary for performance replicaztion, assign it + Create the necessary superuser auth policy necessary for performance replication, assign it to a our previously create test user, and enable performance replication on the primary cluster. EOF @@ -731,10 +731,11 @@ scenario "replication" { ] variables { - primary_leader_public_ip = step.get_primary_cluster_ips.leader_public_ip - vault_addr = step.create_primary_cluster.api_addr_localhost - vault_install_dir = global.vault_install_dir[matrix.artifact_type] - vault_root_token = step.create_primary_cluster.root_token + ip_version = matrix.ip_version + primary_leader_host = step.get_primary_cluster_ips.leader_host + vault_addr = step.create_primary_cluster.api_addr_localhost + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + vault_root_token = step.create_primary_cluster.root_token } } @@ -754,10 +755,12 @@ scenario "replication" { } variables { - primary_leader_public_ip = step.get_primary_cluster_ips.leader_public_ip - vault_addr = step.create_primary_cluster.api_addr_localhost - vault_install_dir = global.vault_install_dir[matrix.artifact_type] - vault_root_token = step.create_primary_cluster.root_token + ip_version = matrix.ip_version + primary_leader_host = step.get_primary_cluster_ips.leader_host + replication_type = "performance" + vault_addr = step.create_primary_cluster.api_addr_localhost + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + vault_root_token = step.create_primary_cluster.root_token } } @@ -766,7 +769,7 @@ scenario "replication" { Enable performance replication on the secondary cluster with the wrapping token created by the primary cluster. EOF - module = module.vault_setup_perf_secondary + module = module.vault_setup_replication_secondary depends_on = [step.generate_secondary_token] providers = { @@ -776,11 +779,13 @@ scenario "replication" { verifies = quality.vault_api_sys_replication_performance_secondary_enable_write variables { - secondary_leader_public_ip = step.get_secondary_cluster_ips.leader_public_ip - vault_addr = step.create_secondary_cluster.api_addr_localhost - vault_install_dir = global.vault_install_dir[matrix.artifact_type] - vault_root_token = step.create_secondary_cluster.root_token - wrapping_token = step.generate_secondary_token.secondary_token + ip_version = matrix.ip_version + secondary_leader_host = step.get_secondary_cluster_ips.leader_host + replication_type = "performance" + vault_addr = step.create_secondary_cluster.api_addr_localhost + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + vault_root_token = step.create_secondary_cluster.root_token + wrapping_token = step.generate_secondary_token.secondary_token } } diff --git a/enos/modules/generate_dr_operation_token/main.tf b/enos/modules/generate_dr_operation_token/main.tf new file mode 100644 index 0000000000..c582c0c73a --- /dev/null +++ b/enos/modules/generate_dr_operation_token/main.tf @@ -0,0 +1,82 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +terraform { + required_providers { + enos = { + source = "registry.terraform.io/hashicorp-forge/enos" + } + random = { + source = "hashicorp/random" + version = ">= 3.4.3" + } + } +} + +variable "ip_version" { + type = number + description = "The IP version used for the Vault TCP listener" + + validation { + condition = contains([4, 6], var.ip_version) + error_message = "The ip_version must be either 4 or 6" + } +} + +variable "primary_leader_host" { + type = object({ + ipv6 = string + private_ip = string + public_ip = string + }) + description = "The primary cluster leader host" +} + +variable "vault_addr" { + type = string + description = "The local vault API listen address" +} + +variable "vault_install_dir" { + type = string + description = "The directory where the Vault binary will be installed" +} + +variable "vault_root_token" { + type = string + description = "The vault root token" +} + +variable "storage_backend" { + type = string + description = "The storage backend to use for the Vault cluster" +} + +locals { + token_id = random_uuid.token_id.id + dr_operation_token = enos_remote_exec.fetch_dr_operation_token.stdout +} + +resource "random_uuid" "token_id" {} + +resource "enos_remote_exec" "fetch_dr_operation_token" { + depends_on = [random_uuid.token_id] + environment = { + VAULT_ADDR = var.vault_addr + VAULT_INSTALL_DIR = var.vault_install_dir + VAULT_TOKEN = var.vault_root_token + STORAGE_BACKEND = var.storage_backend + } + + scripts = [abspath("${path.module}/scripts/configure-vault-dr-primary.sh")] + + transport = { + ssh = { + host = var.primary_leader_host.public_ip + } + } +} + +output "dr_operation_token" { + value = local.dr_operation_token +} diff --git a/enos/modules/generate_dr_operation_token/scripts/configure-vault-dr-primary.sh b/enos/modules/generate_dr_operation_token/scripts/configure-vault-dr-primary.sh new file mode 100755 index 0000000000..eae9b105c2 --- /dev/null +++ b/enos/modules/generate_dr_operation_token/scripts/configure-vault-dr-primary.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +set -e + +binpath="${VAULT_INSTALL_DIR}/vault" + +fail() { + echo "$1" >&2 + exit 1 +} + +# Check required environment variables +[[ -z "$VAULT_ADDR" ]] && fail "VAULT_ADDR env variable has not been set" +[[ -z "$VAULT_INSTALL_DIR" ]] && fail "VAULT_INSTALL_DIR env variable has not been set" +[[ -z "$STORAGE_BACKEND" ]] && fail "STORAGE_BACKEND env variable has not been set" + +# Define the policy content +policy_content() { + cat << EOF +path "sys/replication/dr/secondary/promote" { + capabilities = [ "update" ] +} + +path "sys/replication/dr/secondary/update-primary" { + capabilities = [ "update" ] +} +EOF + if [ "$STORAGE_BACKEND" = "raft" ]; then + cat << EOF +path "sys/storage/raft/autopilot/state" { + capabilities = [ "update", "read" ] +} +EOF + fi +} + +# Write the policy +$binpath policy write dr-secondary-promotion - <<< "$(policy_content)" &> /dev/null + +# Configure the failover handler token role +$binpath write auth/token/roles/failover-handler \ + allowed_policies=dr-secondary-promotion \ + orphan=true \ + renewable=false \ + token_type=batch &> /dev/null + +# Create a token for the failover handler role and output the token only +$binpath token create -field=token -role=failover-handler -ttl=8h diff --git a/enos/modules/generate_failover_secondary_token/main.tf b/enos/modules/generate_failover_secondary_token/main.tf new file mode 100644 index 0000000000..537b0afeb7 --- /dev/null +++ b/enos/modules/generate_failover_secondary_token/main.tf @@ -0,0 +1,98 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +terraform { + required_providers { + enos = { + source = "registry.terraform.io/hashicorp-forge/enos" + } + random = { + source = "hashicorp/random" + version = ">= 3.4.3" + } + } +} + +variable "ip_version" { + type = number + description = "The IP version used for the Vault TCP listener" + + validation { + condition = contains([4, 6], var.ip_version) + error_message = "The ip_version must be either 4 or 6" + } +} + +variable "primary_leader_host" { + type = object({ + ipv6 = string + private_ip = string + public_ip = string + }) + description = "The primary cluster leader host" +} + +variable "retry_interval" { + type = string + default = "2" + description = "How long to wait between retries" +} + +variable "secondary_public_key" { + type = string + description = "The secondary public key" +} + +variable "timeout" { + type = string + default = "15" + description = "How many seconds to wait before timing out" +} + +variable "vault_addr" { + type = string + description = "The local vault API listen address" +} + +variable "vault_install_dir" { + type = string + description = "The directory where the Vault binary will be installed" +} + +variable "vault_root_token" { + type = string + description = "The vault root token" +} + +locals { + primary_leader_addr = var.ip_version == 6 ? var.primary_leader_host.ipv6 : var.primary_leader_host.private_ip + token_id = random_uuid.token_id.id + secondary_token = enos_remote_exec.fetch_secondary_token.stdout +} + +resource "random_uuid" "token_id" {} + +resource "enos_remote_exec" "fetch_secondary_token" { + depends_on = [random_uuid.token_id] + environment = { + VAULT_ADDR = var.vault_addr + VAULT_TOKEN = var.vault_root_token + RETRY_INTERVAL = var.retry_interval + TIMEOUT_SECONDS = var.timeout + SECONDARY_PUBLIC_KEY = var.secondary_public_key + VAULT_INSTALL_DIR = var.vault_install_dir + } + + scripts = [abspath("${path.module}/scripts/generate-failover-secondary-token.sh")] + + + transport = { + ssh = { + host = var.primary_leader_host.public_ip + } + } +} + +output "secondary_token" { + value = local.secondary_token +} diff --git a/enos/modules/generate_failover_secondary_token/scripts/generate-failover-secondary-token.sh b/enos/modules/generate_failover_secondary_token/scripts/generate-failover-secondary-token.sh new file mode 100644 index 0000000000..e8e0e3094b --- /dev/null +++ b/enos/modules/generate_failover_secondary_token/scripts/generate-failover-secondary-token.sh @@ -0,0 +1,33 @@ +#!/bin/bash +## Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +set -e + +[[ -z "${VAULT_INSTALL_DIR}" ]] && fail "VAULT_INSTALL_DIR env variable has not been set" +[[ -z "${RETRY_INTERVAL}" ]] && fail "RETRY_INTERVAL env variable has not been set" +[[ -z "${TIMEOUT_SECONDS}" ]] && fail "TIMEOUT_SECONDS env variable has not been set" +[[ -z "${VAULT_ADDR}" ]] && fail "VAULT_ADDR env variable has not been set" +[[ -z "${VAULT_TOKEN}" ]] && fail "VAULT_TOKEN env variable has not been set" +[[ -z "${SECONDARY_PUBLIC_KEY}" ]] && fail "SECONDARY_PUBLIC_KEY env variable has not been set" + +fail() { + echo "$1" 1>&2 + exit 1 +} + +binpath="${VAULT_INSTALL_DIR}"/vault +test -x "${binpath}" || fail "unable to locate vault binary at ${binpath}" + +begin_time=$(date +%s) +end_time=$((begin_time + TIMEOUT_SECONDS)) +while [ "$(date +%s)" -lt "${end_time}" ]; do + if secondary_token=$(${binpath} write -field token sys/replication/dr/primary/secondary-token id="${VAULT_TOKEN}" secondary_public_key="${SECONDARY_PUBLIC_KEY}"); then + echo "${secondary_token}" + exit 0 + fi + + sleep "${RETRY_INTERVAL}" +done + +fail "Timed out trying to generate secondary token" diff --git a/enos/modules/generate_secondary_public_key/main.tf b/enos/modules/generate_secondary_public_key/main.tf new file mode 100644 index 0000000000..761972dfad --- /dev/null +++ b/enos/modules/generate_secondary_public_key/main.tf @@ -0,0 +1,77 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +terraform { + required_providers { + enos = { + source = "registry.terraform.io/hashicorp-forge/enos" + } + random = { + source = "hashicorp/random" + version = ">= 3.4.3" + } + } +} + +variable "ip_version" { + type = number + description = "The IP version used for the Vault TCP listener" + + validation { + condition = contains([4, 6], var.ip_version) + error_message = "The ip_version must be either 4 or 6" + } +} + +variable "primary_leader_host" { + type = object({ + ipv6 = string + private_ip = string + public_ip = string + }) + description = "The primary cluster leader host" +} + +variable "vault_addr" { + type = string + description = "The local vault API listen address" +} + +variable "vault_install_dir" { + type = string + description = "The directory where the Vault binary will be installed" +} + + +variable "vault_root_token" { + type = string + description = "The vault root token" +} + +locals { + primary_leader_addr = var.ip_version == 6 ? var.primary_leader_host.ipv6 : var.primary_leader_host.private_ip + token_id = random_uuid.token_id.id + secondary_public_key = enos_remote_exec.fetch_secondary_public_key.stdout +} + +resource "random_uuid" "token_id" {} + +resource "enos_remote_exec" "fetch_secondary_public_key" { + depends_on = [random_uuid.token_id] + environment = { + VAULT_ADDR = var.vault_addr + VAULT_TOKEN = var.vault_root_token + } + + inline = ["${var.vault_install_dir}/vault write -field secondary_public_key -f sys/replication/dr/secondary/generate-public-key"] + + transport = { + ssh = { + host = var.primary_leader_host.public_ip + } + } +} + +output "secondary_public_key" { + value = local.secondary_public_key +} diff --git a/enos/modules/generate_secondary_token/main.tf b/enos/modules/generate_secondary_token/main.tf index e7eb3f2123..41b2774e3b 100644 --- a/enos/modules/generate_secondary_token/main.tf +++ b/enos/modules/generate_secondary_token/main.tf @@ -13,6 +13,35 @@ terraform { } } +variable "ip_version" { + type = number + description = "The IP version used for the Vault TCP listener" + + validation { + condition = contains([4, 6], var.ip_version) + error_message = "The ip_version must be either 4 or 6" + } +} + +variable "primary_leader_host" { + type = object({ + ipv6 = string + private_ip = string + public_ip = string + }) + description = "The primary cluster leader host" +} + +variable "replication_type" { + type = string + description = "The type of replication to perform" + + validation { + condition = contains(["dr", "performance"], var.replication_type) + error_message = "The replication_type must be either dr or performance" + } +} + variable "vault_addr" { type = string description = "The local vault API listen address" @@ -23,19 +52,15 @@ variable "vault_install_dir" { description = "The directory where the Vault binary will be installed" } -variable "primary_leader_public_ip" { - type = string - description = "Vault primary cluster leader Public IP address" -} - variable "vault_root_token" { type = string description = "The vault root token" } locals { - token_id = random_uuid.token_id.id - secondary_token = enos_remote_exec.fetch_secondary_token.stdout + primary_leader_addr = var.ip_version == 6 ? var.primary_leader_host.ipv6 : var.primary_leader_host.private_ip + token_id = random_uuid.token_id.id + secondary_token = enos_remote_exec.fetch_secondary_token.stdout } resource "random_uuid" "token_id" {} @@ -47,11 +72,11 @@ resource "enos_remote_exec" "fetch_secondary_token" { VAULT_TOKEN = var.vault_root_token } - inline = ["${var.vault_install_dir}/vault write sys/replication/performance/primary/secondary-token id=${local.token_id} |sed -n '/^wrapping_token:/p' |awk '{print $2}'"] + inline = ["${var.vault_install_dir}/vault write sys/replication/${var.replication_type}/primary/secondary-token id=${local.token_id} |sed -n '/^wrapping_token:/p' |awk '{print $2}'"] transport = { ssh = { - host = var.primary_leader_public_ip + host = var.primary_leader_host.public_ip } } } diff --git a/enos/modules/vault_failover_demote_dr_primary/main.tf b/enos/modules/vault_failover_demote_dr_primary/main.tf new file mode 100644 index 0000000000..819337074f --- /dev/null +++ b/enos/modules/vault_failover_demote_dr_primary/main.tf @@ -0,0 +1,63 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +terraform { + required_providers { + enos = { + source = "registry.terraform.io/hashicorp-forge/enos" + } + } +} + +variable "ip_version" { + type = number + description = "The IP version used for the Vault TCP listener" + + validation { + condition = contains([4, 6], var.ip_version) + error_message = "The ip_version must be either 4 or 6" + } +} + +variable "primary_leader_host" { + type = object({ + ipv6 = string + private_ip = string + public_ip = string + }) + description = "The primary cluster leader host" +} + +variable "vault_install_dir" { + type = string + description = "The directory where the Vault binary will be installed" +} + +variable "vault_addr" { + type = string + description = "The local vault API listen address" +} + +variable "vault_root_token" { + type = string + description = "The vault root token" +} + +locals { +} + +resource "enos_remote_exec" "demote_dr_primary" { + environment = { + VAULT_ADDR = var.vault_addr + VAULT_TOKEN = var.vault_root_token + } + + inline = ["${var.vault_install_dir}/vault write -f sys/replication/dr/primary/demote"] + + + transport = { + ssh = { + host = var.primary_leader_host.public_ip + } + } +} diff --git a/enos/modules/vault_failover_promote_dr_secondary/main.tf b/enos/modules/vault_failover_promote_dr_secondary/main.tf new file mode 100644 index 0000000000..8538253630 --- /dev/null +++ b/enos/modules/vault_failover_promote_dr_secondary/main.tf @@ -0,0 +1,69 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +terraform { + required_providers { + enos = { + source = "registry.terraform.io/hashicorp-forge/enos" + } + } +} + +variable "ip_version" { + type = number + description = "The IP version used for the Vault TCP listener" + + validation { + condition = contains([4, 6], var.ip_version) + error_message = "The ip_version must be either 4 or 6" + } +} + +variable "secondary_leader_host" { + type = object({ + ipv6 = string + private_ip = string + public_ip = string + }) + description = "The secondary cluster leader host" +} + +variable "vault_addr" { + type = string + description = "The local vault API listen address" +} + +variable "vault_install_dir" { + type = string + description = "The directory where the Vault binary will be installed" +} + +variable "vault_root_token" { + type = string + description = "The vault root token" +} + +variable "dr_operation_token" { + type = string + description = "The wrapping token created on primary cluster" +} + +locals { + dr_operation_token = var.dr_operation_token +} + +resource "enos_remote_exec" "promote_dr_secondary" { + environment = { + VAULT_ADDR = var.vault_addr + VAULT_TOKEN = var.vault_root_token + } + + inline = ["${var.vault_install_dir}/vault write -f sys/replication/dr/secondary/promote dr_operation_token=${local.dr_operation_token}"] + + + transport = { + ssh = { + host = var.secondary_leader_host.public_ip + } + } +} diff --git a/enos/modules/vault_failover_update_dr_primary/main.tf b/enos/modules/vault_failover_update_dr_primary/main.tf new file mode 100644 index 0000000000..cc159f2ddd --- /dev/null +++ b/enos/modules/vault_failover_update_dr_primary/main.tf @@ -0,0 +1,76 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +terraform { + required_providers { + enos = { + source = "registry.terraform.io/hashicorp-forge/enos" + } + } +} + +variable "ip_version" { + type = number + description = "The IP version used for the Vault TCP listener" + + validation { + condition = contains([4, 6], var.ip_version) + error_message = "The ip_version must be either 4 or 6" + } +} + +variable "secondary_leader_host" { + type = object({ + ipv6 = string + private_ip = string + public_ip = string + }) + description = "The secondary cluster leader host" +} + +variable "vault_install_dir" { + type = string + description = "The directory where the Vault binary will be installed" +} + +variable "vault_addr" { + type = string + description = "The local vault API listen address" + +} + +variable "vault_root_token" { + type = string + description = "The vault root token" +} + +variable "dr_operation_token" { + type = string + description = "The wrapping token created on primary cluster" +} + +variable "wrapping_token" { + type = string + description = "The wrapping token created on primary cluster" +} + +locals { + dr_operation_token = var.dr_operation_token + wrapping_token = var.wrapping_token +} + +resource "enos_remote_exec" "update_dr_primary" { + environment = { + VAULT_ADDR = var.vault_addr + VAULT_TOKEN = var.vault_root_token + } + + inline = ["${var.vault_install_dir}/vault write sys/replication/dr/secondary/update-primary dr_operation_token=${local.dr_operation_token} token=${local.wrapping_token}"] + + + transport = { + ssh = { + host = var.secondary_leader_host.public_ip + } + } +} diff --git a/enos/modules/vault_setup_dr_primary/main.tf b/enos/modules/vault_setup_dr_primary/main.tf new file mode 100644 index 0000000000..440517b6ed --- /dev/null +++ b/enos/modules/vault_setup_dr_primary/main.tf @@ -0,0 +1,59 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +terraform { + required_providers { + enos = { + source = "registry.terraform.io/hashicorp-forge/enos" + } + } +} + +variable "ip_version" { + type = number + description = "The IP version used for the Vault TCP listener" + + validation { + condition = contains([4, 6], var.ip_version) + error_message = "The ip_version must be either 4 or 6" + } +} + +variable "primary_leader_host" { + type = object({ + ipv6 = string + private_ip = string + public_ip = string + }) + description = "The primary cluster leader host" +} + +variable "vault_addr" { + type = string + description = "The local vault API listen address" +} + +variable "vault_install_dir" { + type = string + description = "The directory where the Vault binary will be installed" +} + +variable "vault_root_token" { + type = string + description = "The vault root token" +} +resource "enos_remote_exec" "configure_dr_primary" { + environment = { + VAULT_ADDR = var.vault_addr + VAULT_TOKEN = var.vault_root_token + VAULT_INSTALL_DIR = var.vault_install_dir + } + + scripts = [abspath("${path.module}/scripts/configure-vault-dr-primary.sh")] + + transport = { + ssh = { + host = var.primary_leader_host.public_ip + } + } +} diff --git a/enos/modules/vault_setup_dr_primary/scripts/configure-vault-dr-primary.sh b/enos/modules/vault_setup_dr_primary/scripts/configure-vault-dr-primary.sh new file mode 100644 index 0000000000..b8c987bc0d --- /dev/null +++ b/enos/modules/vault_setup_dr_primary/scripts/configure-vault-dr-primary.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +set -e + +binpath=${VAULT_INSTALL_DIR}/vault + +fail() { + echo "$1" 1>&2 + return 1 +} + +test -x "$binpath" || fail "unable to locate vault binary at $binpath" + +# Activate the primary +$binpath write -f sys/replication/dr/primary/enable diff --git a/enos/modules/vault_setup_perf_primary/main.tf b/enos/modules/vault_setup_perf_primary/main.tf index d85f7ef6ec..155ab20f4a 100644 --- a/enos/modules/vault_setup_perf_primary/main.tf +++ b/enos/modules/vault_setup_perf_primary/main.tf @@ -9,9 +9,23 @@ terraform { } } -variable "primary_leader_public_ip" { - type = string - description = "Vault primary cluster leader Public IP address" +variable "ip_version" { + type = number + description = "The IP version used for the Vault TCP listener" + + validation { + condition = contains([4, 6], var.ip_version) + error_message = "The ip_version must be either 4 or 6" + } +} + +variable "primary_leader_host" { + type = object({ + ipv6 = string + private_ip = string + public_ip = string + }) + description = "The primary cluster leader host" } variable "vault_addr" { @@ -40,7 +54,7 @@ resource "enos_remote_exec" "configure_pr_primary" { transport = { ssh = { - host = var.primary_leader_public_ip + host = var.primary_leader_host.public_ip } } } diff --git a/enos/modules/vault_setup_perf_primary/scripts/configure-vault-pr-primary.sh b/enos/modules/vault_setup_perf_primary/scripts/configure-vault-pr-primary.sh index 40666abc79..10398b8052 100644 --- a/enos/modules/vault_setup_perf_primary/scripts/configure-vault-pr-primary.sh +++ b/enos/modules/vault_setup_perf_primary/scripts/configure-vault-pr-primary.sh @@ -13,17 +13,5 @@ fail() { test -x "$binpath" || fail "unable to locate vault binary at $binpath" -# Create superuser policy -$binpath policy write superuser - << EOF -path "*" { - capabilities = ["create", "read", "update", "delete", "list", "sudo"] -} -EOF - -# The userpass auth method is enabled with the `vault_verify_write_data`, -# so we do not enable here. -# Create new user and attach superuser policy -$binpath write auth/userpass/users/tester password="changeme" policies="superuser" - # Activate the primary $binpath write -f sys/replication/performance/primary/enable diff --git a/enos/modules/vault_setup_perf_secondary/main.tf b/enos/modules/vault_setup_replication_secondary/main.tf similarity index 53% rename from enos/modules/vault_setup_perf_secondary/main.tf rename to enos/modules/vault_setup_replication_secondary/main.tf index fe2f75511d..fd891e3555 100644 --- a/enos/modules/vault_setup_perf_secondary/main.tf +++ b/enos/modules/vault_setup_replication_secondary/main.tf @@ -9,9 +9,33 @@ terraform { } } -variable "secondary_leader_public_ip" { +variable "ip_version" { + type = number + description = "The IP version used for the Vault TCP listener" + + validation { + condition = contains([4, 6], var.ip_version) + error_message = "The ip_version must be either 4 or 6" + } +} + +variable "secondary_leader_host" { + type = object({ + ipv6 = string + private_ip = string + public_ip = string + }) + description = "The secondary cluster leader host" +} + +variable "replication_type" { type = string - description = "Vault secondary cluster leader Public IP address" + description = "The type of replication to perform" + + validation { + condition = contains(["dr", "performance"], var.replication_type) + error_message = "The replication_type must be either dr or performance" + } } variable "vault_addr" { @@ -40,11 +64,11 @@ resource "enos_remote_exec" "configure_pr_secondary" { VAULT_TOKEN = var.vault_root_token } - inline = ["${var.vault_install_dir}/vault write sys/replication/performance/secondary/enable token=${var.wrapping_token}"] + inline = ["${var.vault_install_dir}/vault write sys/replication/${var.replication_type}/secondary/enable token=${var.wrapping_token}"] transport = { ssh = { - host = var.secondary_leader_public_ip + host = var.secondary_leader_host.public_ip } } } diff --git a/enos/modules/vault_verify_dr_replication/main.tf b/enos/modules/vault_verify_dr_replication/main.tf new file mode 100644 index 0000000000..f7f99fdede --- /dev/null +++ b/enos/modules/vault_verify_dr_replication/main.tf @@ -0,0 +1,117 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +terraform { + required_providers { + enos = { + source = "registry.terraform.io/hashicorp-forge/enos" + } + } +} + +variable "ip_version" { + type = number + description = "The IP version used for the Vault TCP listener" + + validation { + condition = contains([4, 6], var.ip_version) + error_message = "The ip_version must be either 4 or 6" + } +} + +variable "primary_leader_host" { + type = object({ + ipv6 = string + private_ip = string + public_ip = string + }) + description = "The primary cluster leader host" +} + +variable "secondary_leader_host" { + type = object({ + ipv6 = string + private_ip = string + public_ip = string + }) + description = "The secondary cluster leader host" +} + +variable "vault_addr" { + type = string + description = "The local vault API listen address" +} + +variable "vault_install_dir" { + type = string + description = "The directory where the Vault binary will be installed" +} + +variable "wrapping_token" { + type = string + description = "The wrapping token created on primary cluster" + default = null +} + +locals { + primary_leader_addr = var.ip_version == 6 ? var.primary_leader_host.ipv6 : var.primary_leader_host.private_ip + secondary_leader_addr = var.ip_version == 6 ? var.secondary_leader_host.ipv6 : var.secondary_leader_host.private_ip + primary_replication_status = jsondecode(enos_remote_exec.verify_replication_status_on_primary.stdout) + secondary_replication_status = jsondecode(enos_remote_exec.verify_replication_status_on_secondary.stdout) +} + +resource "enos_remote_exec" "verify_replication_status_on_primary" { + environment = { + IP_VERSION = var.ip_version + PRIMARY_LEADER_ADDR = local.primary_leader_addr + SECONDARY_LEADER_ADDR = local.secondary_leader_addr + VAULT_ADDR = var.vault_addr + VAULT_INSTALL_DIR = var.vault_install_dir + } + + scripts = [abspath("${path.module}/scripts/verify-replication-status.sh")] + + transport = { + ssh = { + host = var.primary_leader_host.public_ip + } + } +} + +resource "enos_remote_exec" "verify_replication_status_on_secondary" { + environment = { + IP_VERSION = var.ip_version + PRIMARY_LEADER_ADDR = local.primary_leader_addr + SECONDARY_LEADER_ADDR = local.secondary_leader_addr + VAULT_ADDR = var.vault_addr + VAULT_INSTALL_DIR = var.vault_install_dir + } + + scripts = [abspath("${path.module}/scripts/verify-replication-status.sh")] + + transport = { + ssh = { + host = var.secondary_leader_host.public_ip + } + } +} + +output "primary_replication_status" { + value = local.primary_replication_status +} + +output "known_primary_cluster_addrs" { + value = local.secondary_replication_status.data.known_primary_cluster_addrs +} + +output "secondary_replication_status" { + value = local.secondary_replication_status +} + +output "primary_replication_data_secondaries" { + value = local.primary_replication_status.data.secondaries +} + +output "secondary_replication_data_primaries" { + value = local.secondary_replication_status.data.primaries +} diff --git a/enos/modules/vault_verify_dr_replication/scripts/verify-replication-status.sh b/enos/modules/vault_verify_dr_replication/scripts/verify-replication-status.sh new file mode 100644 index 0000000000..f01a9cd281 --- /dev/null +++ b/enos/modules/vault_verify_dr_replication/scripts/verify-replication-status.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +# This script waits for the replication status to be established +# then verifies the dr replication between primary and +# secondary clusters + +set -e + +fail() { + echo "$1" 1>&2 + exit 1 +} + +[[ -z "$IP_VERSION" ]] && fail "IP_VERSION env variable has not been set" +[[ -z "$PRIMARY_LEADER_ADDR" ]] && fail "PRIMARY_LEADER_ADDR env variable has not been set" +[[ -z "$SECONDARY_LEADER_ADDR" ]] && fail "SECONDARY_LEADER_ADDR env variable has not been set" +[[ -z "$VAULT_ADDR" ]] && fail "VAULT_ADDR env variable has not been set" +[[ -z "$VAULT_INSTALL_DIR" ]] && fail "VAULT_INSTALL_DIR env variable has not been set" + +binpath=${VAULT_INSTALL_DIR}/vault +test -x "$binpath" || fail "unable to locate vault binary at $binpath" + +retry() { + local retries=$1 + shift + local count=0 + + until "$@"; do + wait=$((2 ** count)) + count=$((count + 1)) + if [ "$count" -lt "$retries" ]; then + sleep "$wait" + else + fail "$($binpath read -format=json sys/replication/dr/status)" + fi + done +} + +check_dr_status() { + dr_status=$($binpath read -format=json sys/replication/dr/status) + cluster_state=$(jq -r '.data.state' <<< "$dr_status") + connection_mode=$(jq -r '.data.mode' <<< "$dr_status") + + if [[ "$cluster_state" == 'idle' ]]; then + echo "replication cluster state is idle" 1>&2 + return 1 + fi + + if [[ "$connection_mode" == "primary" ]]; then + connection_status=$(jq -r '.data.secondaries[0].connection_status' <<< "$dr_status") + if [[ "$connection_status" == 'disconnected' ]]; then + echo ".data.secondaries[0].connection_status from primary node is 'disconnected'" 1>&2 + return 1 + fi + # Confirm we are in a "running" state for the primary + if [[ "$cluster_state" != "running" ]]; then + echo "replication cluster primary state is not running" 1>&2 + return 1 + fi + else + connection_status=$(jq -r '.data.primaries[0].connection_status' <<< "$dr_status") + if [[ "$connection_status" == 'disconnected' ]]; then + echo ".data.primaries[0].connection_status from secondary node is 'disconnected'" 1>&2 + return 1 + fi + # Confirm we are in a "stream-wals" state for the secondary + if [[ "$cluster_state" != "stream-wals" ]]; then + echo "replication cluster primary state is not stream-wals" 1>&2 + return 1 + fi + known_primary_cluster_addrs=$(jq -r '.data.known_primary_cluster_addrs' <<< "$dr_status") + if ! echo "$known_primary_cluster_addrs" | grep -q "$PRIMARY_LEADER_ADDR"; then + echo "$PRIMARY_LEADER_ADDR is not in .data.known_primary_cluster_addrs: $known_primary_cluster_addrs" 1>&2 + return 1 + fi + fi + + echo "$dr_status" + return 0 +} + +if [ "$IP_VERSION" != 4 ] && [ "$IP_VERSION" != 6 ]; then + fail "unsupported IP_VERSION: $IP_VERSION" +fi + +# Retry for a while because it can take some time for replication to sync +retry 10 check_dr_status