vault/enos/enos-scenario-proxy.hcl
Ryan Cragun 1082629d1f
VAULT-30819: Fix two potential flakes in DR replication (#28409)
Fix two occasional flakes in the DR replication scenario:
* Always verify that all nodes in the cluster are unsealed before
  verifying test data. Previously we only verified seal status on
  followers.
* Fix an occasional timeout when waiting for the cluster to unseal by
  rewriting the module to retry for a set duration instead of
  exponential backoff.

Signed-off-by: Ryan Cragun <me@ryan.ec>
2024-09-17 12:32:15 -06:00

626 lines
21 KiB
HCL

// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: BUSL-1.1
scenario "proxy" {
description = <<-EOF
The agent scenario verifies Vault when running in Proxy mode. The build can be a local branch,
any CRT built Vault artifact saved to the local machine, or any CRT built Vault artifact in the
stable channel in Artifactory.
The scenario creates a new Vault Cluster using the candidate build and then runs the same Vault
build in Proxy mode and verifies behavior against the Vault cluster. The scenario also performs
standard baseline verification that is not specific to the Proxy mode deployment.
If you want to use the 'distro:leap' variant you must first accept SUSE's terms for the AWS
account. To verify that your account has agreed, sign-in to your AWS through Doormat,
and visit the following links to verify your subscription or subscribe:
arm64 AMI: https://aws.amazon.com/marketplace/server/procurement?productId=a516e959-df54-4035-bb1a-63599b7a6df9
amd64 AMI: https://aws.amazon.com/marketplace/server/procurement?productId=5535c495-72d4-4355-b169-54ffa874f849
EOF
matrix {
arch = global.archs
artifact_source = global.artifact_sources
artifact_type = global.artifact_types
backend = global.backends
config_mode = global.config_modes
consul_edition = global.consul_editions
consul_version = global.consul_versions
distro = global.distros
edition = global.editions
ip_version = global.ip_versions
seal = global.seals
// Our local builder always creates bundles
exclude {
artifact_source = ["local"]
artifact_type = ["package"]
}
// PKCS#11 can only be used on ent.hsm and ent.hsm.fips1402.
exclude {
seal = ["pkcs11"]
edition = [for e in matrix.edition : e if !strcontains(e, "hsm")]
}
// softhsm packages not available for leap/sles.
exclude {
seal = ["pkcs11"]
distro = ["leap", "sles"]
}
// Testing in IPV6 mode is currently implemented for integrated Raft storage only
exclude {
ip_version = ["6"]
backend = ["consul"]
}
}
terraform_cli = terraform_cli.default
terraform = terraform.default
providers = [
provider.aws.default,
provider.enos.ec2_user,
provider.enos.ubuntu
]
locals {
artifact_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_artifact_path) : null
enos_provider = {
amzn = provider.enos.ec2_user
leap = provider.enos.ec2_user
rhel = provider.enos.ec2_user
sles = provider.enos.ec2_user
ubuntu = provider.enos.ubuntu
}
manage_service = matrix.artifact_type == "bundle"
vault_install_dir = global.vault_install_dir[matrix.artifact_type]
}
step "get_local_metadata" {
description = global.description.get_local_metadata
skip_step = matrix.artifact_source != "local"
module = module.get_local_metadata
}
step "build_vault" {
description = global.description.build_vault
module = "build_${matrix.artifact_source}"
variables {
build_tags = var.vault_local_build_tags != null ? var.vault_local_build_tags : global.build_tags[matrix.edition]
artifact_path = local.artifact_path
goarch = matrix.arch
goos = "linux"
artifactory_host = matrix.artifact_source == "artifactory" ? var.artifactory_host : null
artifactory_repo = matrix.artifact_source == "artifactory" ? var.artifactory_repo : null
artifactory_username = matrix.artifact_source == "artifactory" ? var.artifactory_username : null
artifactory_token = matrix.artifact_source == "artifactory" ? var.artifactory_token : null
arch = matrix.artifact_source == "artifactory" ? matrix.arch : null
product_version = var.vault_product_version
artifact_type = matrix.artifact_type
distro = matrix.artifact_source == "artifactory" ? matrix.distro : null
edition = matrix.artifact_source == "artifactory" ? matrix.edition : null
revision = var.vault_revision
}
}
step "ec2_info" {
description = global.description.ec2_info
module = module.ec2_info
}
step "create_vpc" {
description = global.description.create_vpc
module = module.create_vpc
variables {
common_tags = global.tags
ip_version = matrix.ip_version
}
}
// This step reads the contents of the backend license if we're using a Consul backend and
// an "ent" Consul edition.
step "read_backend_license" {
description = global.description.read_backend_license
skip_step = matrix.backend == "raft" || matrix.consul_edition == "ce"
module = module.read_license
variables {
file_name = global.backend_license_path
}
}
step "read_vault_license" {
description = global.description.read_vault_license
skip_step = matrix.edition == "ce"
module = module.read_license
variables {
file_name = global.vault_license_path
}
}
step "create_seal_key" {
description = global.description.create_seal_key
module = "seal_${matrix.seal}"
depends_on = [step.create_vpc]
providers = {
enos = provider.enos.ubuntu
}
variables {
cluster_id = step.create_vpc.id
common_tags = global.tags
}
}
step "create_vault_cluster_targets" {
description = global.description.create_vault_cluster_targets
module = module.target_ec2_instances
depends_on = [step.create_vpc]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]]
cluster_tag_key = global.vault_tag_key
common_tags = global.tags
seal_key_names = step.create_seal_key.resource_names
vpc_id = step.create_vpc.id
}
}
step "create_vault_cluster_backend_targets" {
description = global.description.create_vault_cluster_targets
module = matrix.backend == "consul" ? module.target_ec2_instances : module.target_ec2_shim
depends_on = [step.create_vpc]
providers = {
enos = provider.enos.ubuntu
}
variables {
ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"][global.distro_version["ubuntu"]]
cluster_tag_key = global.backend_tag_key
common_tags = global.tags
seal_key_names = step.create_seal_key.resource_names
vpc_id = step.create_vpc.id
}
}
step "create_backend_cluster" {
description = global.description.create_backend_cluster
module = "backend_${matrix.backend}"
depends_on = [
step.create_vault_cluster_backend_targets
]
providers = {
enos = provider.enos.ubuntu
}
verifies = [
// verified in modules
quality.consul_autojoin_aws,
quality.consul_config_file,
quality.consul_ha_leader_election,
quality.consul_service_start_server,
// verified in enos_consul_start resource
quality.consul_api_agent_host_read,
quality.consul_api_health_node_read,
quality.consul_api_operator_raft_config_read,
quality.consul_cli_validate,
quality.consul_health_state_passing_read_nodes_minimum,
quality.consul_operator_raft_configuration_read_voters_minimum,
quality.consul_service_systemd_notified,
quality.consul_service_systemd_unit,
]
variables {
cluster_name = step.create_vault_cluster_backend_targets.cluster_name
cluster_tag_key = global.backend_tag_key
hosts = step.create_vault_cluster_backend_targets.hosts
license = (matrix.backend == "consul" && matrix.consul_edition == "ent") ? step.read_backend_license.license : null
release = {
edition = matrix.consul_edition
version = matrix.consul_version
}
}
}
step "create_vault_cluster" {
description = global.description.create_vault_cluster
module = module.vault_cluster
depends_on = [
step.create_backend_cluster,
step.build_vault,
step.create_vault_cluster_targets
]
providers = {
enos = local.enos_provider[matrix.distro]
}
verifies = [
// verified in modules
quality.consul_service_start_client,
quality.vault_artifact_bundle,
quality.vault_artifact_deb,
quality.vault_artifact_rpm,
quality.vault_audit_log,
quality.vault_audit_socket,
quality.vault_audit_syslog,
quality.vault_autojoin_aws,
quality.vault_config_env_variables,
quality.vault_config_file,
quality.vault_config_log_level,
quality.vault_init,
quality.vault_license_required_ent,
quality.vault_listener_ipv4,
quality.vault_listener_ipv6,
quality.vault_service_start,
quality.vault_storage_backend_consul,
quality.vault_storage_backend_raft,
// verified in enos_vault_start resource
quality.vault_api_sys_config_read,
quality.vault_api_sys_ha_status_read,
quality.vault_api_sys_health_read,
quality.vault_api_sys_host_info_read,
quality.vault_api_sys_replication_status_read,
quality.vault_api_sys_seal_status_api_read_matches_sys_health,
quality.vault_api_sys_storage_raft_autopilot_configuration_read,
quality.vault_api_sys_storage_raft_autopilot_state_read,
quality.vault_api_sys_storage_raft_configuration_read,
quality.vault_service_systemd_notified,
quality.vault_service_systemd_unit,
quality.vault_cli_status_exit_code,
]
variables {
artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
backend_cluster_name = step.create_vault_cluster_backend_targets.cluster_name
backend_cluster_tag_key = global.backend_tag_key
cluster_name = step.create_vault_cluster_targets.cluster_name
config_mode = matrix.config_mode
consul_license = (matrix.backend == "consul" && matrix.consul_edition == "ent") ? step.read_backend_license.license : null
consul_release = matrix.backend == "consul" ? {
edition = matrix.consul_edition
version = matrix.consul_version
} : null
enable_audit_devices = var.vault_enable_audit_devices
hosts = step.create_vault_cluster_targets.hosts
install_dir = global.vault_install_dir[matrix.artifact_type]
ip_version = matrix.ip_version
license = matrix.edition != "ce" ? step.read_vault_license.license : null
local_artifact_path = local.artifact_path
manage_service = local.manage_service
packages = concat(global.packages, global.distro_packages[matrix.distro][global.distro_version[matrix.distro]])
seal_attributes = step.create_seal_key.attributes
seal_type = matrix.seal
storage_backend = matrix.backend
}
}
// Wait for our cluster to elect a leader
step "wait_for_leader" {
description = global.description.wait_for_cluster_to_have_leader
module = module.vault_wait_for_leader
depends_on = [step.create_vault_cluster]
providers = {
enos = local.enos_provider[matrix.distro]
}
verifies = [
quality.vault_api_sys_leader_read,
quality.vault_unseal_ha_leader_election,
]
variables {
hosts = step.create_vault_cluster_targets.hosts
ip_version = matrix.ip_version
timeout = 120 // seconds
vault_addr = step.create_vault_cluster.api_addr_localhost
vault_install_dir = global.vault_install_dir[matrix.artifact_type]
vault_root_token = step.create_vault_cluster.root_token
}
}
step "start_vault_proxy" {
module = module.vault_proxy
depends_on = [
step.build_vault,
step.create_vault_cluster,
]
providers = {
enos = local.enos_provider[matrix.distro]
}
verifies = [
quality.vault_cli_auth_enable_approle,
quality.vault_proxy_auto_auth_approle,
quality.vault_proxy_cli_access,
]
variables {
hosts = step.create_vault_cluster_targets.hosts
ip_version = matrix.ip_version
vault_addr = step.create_vault_cluster.api_addr_localhost
vault_install_dir = global.vault_install_dir[matrix.artifact_type]
vault_root_token = step.create_vault_cluster.root_token
}
}
step "get_vault_cluster_ips" {
description = global.description.get_vault_cluster_ip_addresses
module = module.vault_get_cluster_ips
depends_on = [step.wait_for_leader]
providers = {
enos = local.enos_provider[matrix.distro]
}
verifies = [
quality.vault_api_sys_ha_status_read,
quality.vault_api_sys_leader_read,
quality.vault_cli_operator_members,
]
variables {
hosts = step.create_vault_cluster_targets.hosts
ip_version = matrix.ip_version
vault_addr = step.create_vault_cluster.api_addr_localhost
vault_install_dir = global.vault_install_dir[matrix.artifact_type]
vault_root_token = step.create_vault_cluster.root_token
}
}
step "verify_vault_unsealed" {
description = global.description.verify_vault_unsealed
module = module.vault_wait_for_cluster_unsealed
depends_on = [step.create_vault_cluster]
providers = {
enos = local.enos_provider[matrix.distro]
}
verifies = [
quality.vault_seal_awskms,
quality.vault_seal_pkcs11,
quality.vault_seal_shamir,
]
variables {
hosts = step.create_vault_cluster_targets.hosts
vault_addr = step.create_vault_cluster.api_addr_localhost
vault_install_dir = global.vault_install_dir[matrix.artifact_type]
}
}
step "verify_vault_version" {
description = global.description.verify_vault_version
module = module.vault_verify_version
depends_on = [step.verify_vault_unsealed]
providers = {
enos = local.enos_provider[matrix.distro]
}
verifies = [
quality.vault_api_sys_version_history_keys,
quality.vault_api_sys_version_history_key_info,
quality.vault_version_build_date,
quality.vault_version_edition,
quality.vault_version_release,
]
variables {
hosts = step.create_vault_cluster_targets.hosts
vault_addr = step.create_vault_cluster.api_addr_localhost
vault_edition = matrix.edition
vault_install_dir = global.vault_install_dir[matrix.artifact_type]
vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version
vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision
vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date
vault_root_token = step.create_vault_cluster.root_token
}
}
step "verify_secrets_engines_create" {
description = global.description.verify_secrets_engines_create
module = module.vault_verify_secrets_engines_create
depends_on = [step.verify_vault_unsealed]
providers = {
enos = local.enos_provider[matrix.distro]
}
verifies = [
quality.vault_api_auth_userpass_login_write,
quality.vault_api_auth_userpass_user_write,
quality.vault_api_identity_entity_write,
quality.vault_api_identity_entity_alias_write,
quality.vault_api_identity_group_write,
quality.vault_api_identity_oidc_config_write,
quality.vault_api_identity_oidc_introspect_write,
quality.vault_api_identity_oidc_key_write,
quality.vault_api_identity_oidc_key_rotate_write,
quality.vault_api_identity_oidc_role_write,
quality.vault_api_identity_oidc_token_read,
quality.vault_api_sys_auth_userpass_user_write,
quality.vault_api_sys_policy_write,
quality.vault_mount_auth,
quality.vault_mount_kv,
quality.vault_secrets_kv_write,
]
variables {
hosts = step.create_vault_cluster_targets.hosts
leader_host = step.get_vault_cluster_ips.leader_host
vault_addr = step.create_vault_cluster.api_addr_localhost
vault_install_dir = global.vault_install_dir[matrix.artifact_type]
vault_root_token = step.create_vault_cluster.root_token
}
}
step "verify_raft_auto_join_voter" {
description = global.description.verify_raft_cluster_all_nodes_are_voters
skip_step = matrix.backend != "raft"
module = module.vault_verify_raft_auto_join_voter
depends_on = [step.verify_vault_unsealed]
providers = {
enos = local.enos_provider[matrix.distro]
}
verifies = quality.vault_raft_voters
variables {
hosts = step.create_vault_cluster_targets.hosts
ip_version = matrix.ip_version
vault_addr = step.create_vault_cluster.api_addr_localhost
vault_install_dir = global.vault_install_dir[matrix.artifact_type]
vault_root_token = step.create_vault_cluster.root_token
}
}
step "verify_replication" {
description = global.description.verify_replication_status
module = module.vault_verify_replication
depends_on = [step.verify_vault_unsealed]
providers = {
enos = local.enos_provider[matrix.distro]
}
verifies = [
quality.vault_replication_ce_disabled,
quality.vault_replication_ent_dr_available,
quality.vault_replication_ent_pr_available,
]
variables {
hosts = step.create_vault_cluster_targets.hosts
vault_addr = step.create_vault_cluster.api_addr_localhost
vault_edition = matrix.edition
}
}
step "verify_secrets_engines_read" {
description = global.description.verify_secrets_engines_read
module = module.vault_verify_secrets_engines_read
depends_on = [
step.verify_secrets_engines_create,
step.verify_replication
]
providers = {
enos = local.enos_provider[matrix.distro]
}
verifies = [
quality.vault_api_auth_userpass_login_write,
quality.vault_api_identity_entity_read,
quality.vault_api_identity_oidc_config_read,
quality.vault_api_identity_oidc_key_read,
quality.vault_api_identity_oidc_role_read,
quality.vault_secrets_kv_read
]
variables {
create_state = step.verify_secrets_engines_create.state
hosts = step.get_vault_cluster_ips.follower_hosts
vault_addr = step.create_vault_cluster.api_addr_localhost
vault_install_dir = global.vault_install_dir[matrix.artifact_type]
}
}
step "verify_ui" {
description = global.description.verify_ui
module = module.vault_verify_ui
depends_on = [step.verify_vault_unsealed]
providers = {
enos = local.enos_provider[matrix.distro]
}
verifies = quality.vault_ui_assets
variables {
hosts = step.create_vault_cluster_targets.hosts
vault_addr = step.create_vault_cluster.api_addr_localhost
}
}
output "audit_device_file_path" {
description = "The file path for the file audit device, if enabled"
value = step.create_vault_cluster.audit_device_file_path
}
output "cluster_name" {
description = "The Vault cluster name"
value = step.create_vault_cluster.cluster_name
}
output "hosts" {
description = "The Vault cluster target hosts"
value = step.create_vault_cluster.hosts
}
output "private_ips" {
description = "The Vault cluster private IPs"
value = step.create_vault_cluster.private_ips
}
output "public_ips" {
description = "The Vault cluster public IPs"
value = step.create_vault_cluster.public_ips
}
output "root_token" {
description = "The Vault cluster root token"
value = step.create_vault_cluster.root_token
}
output "recovery_key_shares" {
description = "The Vault cluster recovery key shares"
value = step.create_vault_cluster.recovery_key_shares
}
output "recovery_keys_b64" {
description = "The Vault cluster recovery keys b64"
value = step.create_vault_cluster.recovery_keys_b64
}
output "recovery_keys_hex" {
description = "The Vault cluster recovery keys hex"
value = step.create_vault_cluster.recovery_keys_hex
}
output "secrets_engines_state" {
description = "The state of configured secrets engines"
value = step.verify_secrets_engines_create.state
}
output "seal_attributes" {
description = "The Vault cluster seal attributes"
value = step.create_seal_key.attributes
}
output "unseal_keys_b64" {
description = "The Vault cluster unseal keys"
value = step.create_vault_cluster.unseal_keys_b64
}
output "unseal_keys_hex" {
description = "The Vault cluster unseal keys hex"
value = step.create_vault_cluster.unseal_keys_hex
}
}