diff --git a/enos/enos-descriptions.hcl b/enos/enos-descriptions.hcl index bc91d74c75..eefba5fd73 100644 --- a/enos/enos-descriptions.hcl +++ b/enos/enos-descriptions.hcl @@ -211,6 +211,18 @@ globals { the Vault version, edition, build date, and any special prerelease metadata. EOF + run_verify_blackbox_tests = <<-EOF + Run blackbox verification tests via the Vault API from the CI/GitHub runner. These tests + validate Vault functionality without requiring direct access to the Vault binary on the + target hosts. + EOF + + run_verify_blackbox_tests_remote = <<-EOF + Run blackbox verification tests directly on the Vault leader host. These tests execute + the Vault CLI binary on the target machine to validate version metadata and other + functionality that requires local binary access. + EOF + wait_for_cluster_to_have_leader = <<-EOF Wait for a leader election to occur before we proceed with any further quality verification. EOF diff --git a/enos/enos-modules.hcl b/enos/enos-modules.hcl index de8579d0ce..ff088a793f 100644 --- a/enos/enos-modules.hcl +++ b/enos/enos-modules.hcl @@ -408,12 +408,6 @@ module "vault_wait_for_cluster_unsealed" { vault_install_dir = var.vault_install_dir } -module "vault_verify_version" { - source = "./modules/vault_verify_version" - - vault_install_dir = var.vault_install_dir -} - module "vault_wait_for_leader" { source = "./modules/vault_wait_for_leader" diff --git a/enos/enos-scenario-agent.hcl b/enos/enos-scenario-agent.hcl index 4a5f7971c3..87285fb4de 100644 --- a/enos/enos-scenario-agent.hcl +++ b/enos/enos-scenario-agent.hcl @@ -483,10 +483,10 @@ scenario "agent" { } } - step "verify_vault_version" { - description = global.description.verify_vault_version - module = module.vault_verify_version - depends_on = [step.verify_vault_unsealed] + step "run_verify_blackbox_tests" { + description = global.description.run_verify_blackbox_tests + module = module.vault_run_blackbox_test + depends_on = [step.verify_vault_unsealed, step.get_vault_cluster_ips] providers = { enos = local.enos_provider[matrix.distro] @@ -501,14 +501,45 @@ scenario "agent" { ] variables { - hosts = step.create_vault_cluster_targets.hosts - vault_addr = step.create_vault_cluster.api_addr_localhost + leader_host = step.get_vault_cluster_ips.leader_host + leader_public_ip = step.get_vault_cluster_ips.leader_public_ip + vault_root_token = step.create_vault_cluster.root_token + test_package = "./vault/external_tests/blackbox/verify" + test_names = ["TestVaultServerVersion"] vault_edition = matrix.edition - vault_install_dir = global.vault_install_dir[matrix.artifact_type] vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + } + } + + step "run_verify_blackbox_tests_remote" { + description = global.description.run_verify_blackbox_tests_remote + module = module.vault_run_blackbox_test + depends_on = [step.run_verify_blackbox_tests] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = [ + quality.vault_version_build_date, + quality.vault_version_edition, + quality.vault_version_release, + ] + + variables { + leader_host = step.get_vault_cluster_ips.leader_host + leader_public_ip = step.get_vault_cluster_ips.leader_public_ip vault_root_token = step.create_vault_cluster.root_token + test_package = "./vault/external_tests/blackbox/verify" + test_names = ["TestVaultCLIVersionLocal"] + vault_edition = matrix.edition + vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version + vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision + vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date + vault_install_dir = global.vault_install_dir[matrix.artifact_type] } } diff --git a/enos/enos-scenario-autopilot.hcl b/enos/enos-scenario-autopilot.hcl index 03a756d8c7..4dc52aa39d 100644 --- a/enos/enos-scenario-autopilot.hcl +++ b/enos/enos-scenario-autopilot.hcl @@ -844,14 +844,15 @@ scenario "autopilot" { } } - step "verify_vault_version" { - description = global.description.verify_vault_version - module = module.vault_verify_version + step "run_verify_blackbox_tests" { + description = global.description.run_verify_blackbox_tests + module = module.vault_run_blackbox_test depends_on = [ step.create_vault_cluster_upgrade_targets, step.upgrade_vault_cluster_with_autopilot, step.verify_raft_auto_join_voter, - step.remove_old_nodes + step.remove_old_nodes, + step.get_updated_vault_cluster_ips ] providers = { @@ -867,14 +868,16 @@ scenario "autopilot" { ] variables { - hosts = step.upgrade_vault_cluster_with_autopilot.hosts - vault_addr = step.upgrade_vault_cluster_with_autopilot.api_addr_localhost + leader_host = step.get_updated_vault_cluster_ips.leader_host + leader_public_ip = step.get_updated_vault_cluster_ips.leader_public_ip + vault_root_token = step.create_vault_cluster.root_token + test_package = "./vault/external_tests/blackbox/verify" + test_names = ["TestVaultServerVersion"] vault_edition = matrix.edition - vault_install_dir = local.vault_install_dir vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date - vault_root_token = step.create_vault_cluster.root_token + vault_install_dir = local.vault_install_dir } } diff --git a/enos/enos-scenario-dr-replication.hcl b/enos/enos-scenario-dr-replication.hcl index 1ccf424f7c..046daff1ee 100644 --- a/enos/enos-scenario-dr-replication.hcl +++ b/enos/enos-scenario-dr-replication.hcl @@ -665,9 +665,9 @@ scenario "dr_replication" { } } - step "verify_vault_version" { - description = global.description.verify_vault_version - module = module.vault_verify_version + step "run_verify_blackbox_tests" { + description = global.description.run_verify_blackbox_tests + module = module.vault_run_blackbox_test depends_on = [step.get_primary_cluster_ips] providers = { @@ -683,14 +683,45 @@ scenario "dr_replication" { ] variables { - hosts = step.create_primary_cluster_targets.hosts - vault_addr = step.create_primary_cluster.api_addr_localhost + leader_host = step.get_primary_cluster_ips.leader_host + leader_public_ip = step.get_primary_cluster_ips.leader_public_ip + vault_root_token = step.create_primary_cluster.root_token + test_package = "./vault/external_tests/blackbox/verify" + test_names = ["TestVaultServerVersion"] vault_edition = matrix.edition - vault_install_dir = global.vault_install_dir[matrix.artifact_type] vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + } + } + + step "run_verify_blackbox_tests_remote" { + description = global.description.run_verify_blackbox_tests_remote + module = module.vault_run_blackbox_test + depends_on = [step.run_verify_blackbox_tests] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = [ + quality.vault_version_build_date, + quality.vault_version_edition, + quality.vault_version_release, + ] + + variables { + leader_host = step.get_primary_cluster_ips.leader_host + leader_public_ip = step.get_primary_cluster_ips.leader_public_ip vault_root_token = step.create_primary_cluster.root_token + test_package = "./vault/external_tests/blackbox/verify" + test_names = ["TestVaultCLIVersionLocal"] + vault_edition = matrix.edition + vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version + vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision + vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date + vault_install_dir = global.vault_install_dir[matrix.artifact_type] } } diff --git a/enos/enos-scenario-plugin.hcl b/enos/enos-scenario-plugin.hcl index 2563cf01a7..23291ffb8d 100644 --- a/enos/enos-scenario-plugin.hcl +++ b/enos/enos-scenario-plugin.hcl @@ -433,10 +433,10 @@ scenario "plugin" { } } - step "verify_vault_version" { - description = global.description.verify_vault_version - module = module.vault_verify_version - depends_on = [step.verify_vault_unsealed] + step "run_verify_blackbox_tests" { + description = global.description.run_verify_blackbox_tests + module = module.vault_run_blackbox_test + depends_on = [step.verify_vault_unsealed, step.get_vault_cluster_ips] providers = { enos = local.enos_provider[matrix.distro] @@ -451,14 +451,47 @@ scenario "plugin" { ] variables { - hosts = step.create_vault_cluster_targets.hosts - vault_addr = step.create_vault_cluster.api_addr_localhost + leader_host = step.get_vault_cluster_ips.leader_host + leader_public_ip = step.get_vault_cluster_ips.leader_public_ip + vault_root_token = step.create_vault_cluster.root_token + test_package = "./vault/external_tests/blackbox/verify" + test_names = ["TestVaultServerVersion"] vault_edition = matrix.edition - vault_install_dir = global.vault_install_dir[matrix.artifact_type] vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + + } + } + + step "run_verify_blackbox_tests_remote" { + description = global.description.run_verify_blackbox_tests_remote + module = module.vault_run_blackbox_test + depends_on = [step.run_verify_blackbox_tests] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = [ + quality.vault_version_build_date, + quality.vault_version_edition, + quality.vault_version_release, + ] + + variables { + leader_host = step.get_vault_cluster_ips.leader_host + leader_public_ip = step.get_vault_cluster_ips.leader_public_ip vault_root_token = step.create_vault_cluster.root_token + test_package = "./vault/external_tests/blackbox/verify" + test_names = ["TestVaultCLIVersionLocal"] + vault_edition = matrix.edition + vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version + vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision + vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + } } @@ -476,14 +509,19 @@ scenario "plugin" { step "run_plugin_blackbox_tests" { description = local.is_test_name_filter ? "Run specific plugin tests: ${join(", ", var.blackbox_test_filter)}" : "Run plugin blackbox tests from: ${join(", ", length(var.blackbox_test_filter) > 0 && !local.is_test_name_filter ? var.blackbox_test_filter : ["plugins"])}" module = module.vault_run_blackbox_test - depends_on = [step.get_vault_cluster_ips, step.set_up_plugin_services, step.verify_vault_version] + depends_on = [step.get_vault_cluster_ips, step.set_up_plugin_services, step.run_verify_blackbox_tests] providers = { enos = local.enos_provider[matrix.distro] } verifies = [ - // Plugin testing quality - tests will define their own verification + quality.vault_secrets_ldap_write_config, + quality.vault_secrets_kmip_write_config, + quality.vault_secrets_kv_read, + quality.vault_secrets_kv_write, + quality.vault_mount_auth, + quality.vault_mount_kv, ] variables { diff --git a/enos/enos-scenario-pr-replication.hcl b/enos/enos-scenario-pr-replication.hcl index 7adbf9e456..d494b94a7c 100644 --- a/enos/enos-scenario-pr-replication.hcl +++ b/enos/enos-scenario-pr-replication.hcl @@ -687,9 +687,9 @@ scenario "pr_replication" { } } - step "verify_vault_version" { - description = global.description.verify_vault_version - module = module.vault_verify_version + step "run_verify_blackbox_tests" { + description = global.description.run_verify_blackbox_tests + module = module.vault_run_blackbox_test depends_on = [step.get_primary_cluster_ips] providers = { @@ -705,14 +705,45 @@ scenario "pr_replication" { ] variables { - hosts = step.create_primary_cluster_targets.hosts - vault_addr = step.create_primary_cluster.api_addr_localhost + leader_host = step.get_primary_cluster_ips.leader_host + leader_public_ip = step.get_primary_cluster_ips.leader_public_ip + vault_root_token = step.create_primary_cluster.root_token + test_package = "./vault/external_tests/blackbox/verify" + test_names = ["TestVaultServerVersion"] vault_edition = matrix.edition - vault_install_dir = global.vault_install_dir[matrix.artifact_type] vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + } + } + + step "run_verify_blackbox_tests_remote" { + description = global.description.run_verify_blackbox_tests_remote + module = module.vault_run_blackbox_test + depends_on = [step.run_verify_blackbox_tests] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = [ + quality.vault_version_build_date, + quality.vault_version_edition, + quality.vault_version_release, + ] + + variables { + leader_host = step.get_primary_cluster_ips.leader_host + leader_public_ip = step.get_primary_cluster_ips.leader_public_ip vault_root_token = step.create_primary_cluster.root_token + test_package = "./vault/external_tests/blackbox/verify" + test_names = ["TestVaultCLIVersionLocal"] + vault_edition = matrix.edition + vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version + vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision + vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date + vault_install_dir = global.vault_install_dir[matrix.artifact_type] } } @@ -794,7 +825,7 @@ scenario "pr_replication" { step.get_secondary_cluster_ips, step.verify_secrets_engines_on_primary, // Wait base verification to complete... - step.verify_vault_version, + step.run_verify_blackbox_tests, step.verify_ui, ] diff --git a/enos/enos-scenario-proxy.hcl b/enos/enos-scenario-proxy.hcl index 3ae62022f0..bdb2538f0a 100644 --- a/enos/enos-scenario-proxy.hcl +++ b/enos/enos-scenario-proxy.hcl @@ -459,10 +459,10 @@ scenario "proxy" { } } - step "verify_vault_version" { - description = global.description.verify_vault_version - module = module.vault_verify_version - depends_on = [step.verify_vault_unsealed] + step "run_verify_blackbox_tests" { + description = global.description.run_verify_blackbox_tests + module = module.vault_run_blackbox_test + depends_on = [step.verify_vault_unsealed, step.get_vault_cluster_ips] providers = { enos = local.enos_provider[matrix.distro] @@ -477,14 +477,47 @@ scenario "proxy" { ] variables { - hosts = step.create_vault_cluster_targets.hosts - vault_addr = step.create_vault_cluster.api_addr_localhost + leader_host = step.get_vault_cluster_ips.leader_host + leader_public_ip = step.get_vault_cluster_ips.leader_public_ip + vault_root_token = step.create_vault_cluster.root_token + test_package = "./vault/external_tests/blackbox/verify" + test_names = ["TestVaultServerVersion"] vault_edition = matrix.edition - vault_install_dir = global.vault_install_dir[matrix.artifact_type] vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + + } + } + + step "run_verify_blackbox_tests_remote" { + description = global.description.run_verify_blackbox_tests_remote + module = module.vault_run_blackbox_test + depends_on = [step.run_verify_blackbox_tests] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = [ + quality.vault_version_build_date, + quality.vault_version_edition, + quality.vault_version_release, + ] + + variables { + leader_host = step.get_vault_cluster_ips.leader_host + leader_public_ip = step.get_vault_cluster_ips.leader_public_ip vault_root_token = step.create_vault_cluster.root_token + test_package = "./vault/external_tests/blackbox/verify" + test_names = ["TestVaultCLIVersionLocal"] + vault_edition = matrix.edition + vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version + vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision + vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + } } diff --git a/enos/enos-scenario-seal-ha.hcl b/enos/enos-scenario-seal-ha.hcl index c421932401..6af3e8e06c 100644 --- a/enos/enos-scenario-seal-ha.hcl +++ b/enos/enos-scenario-seal-ha.hcl @@ -760,10 +760,10 @@ scenario "seal_ha" { } // Perform all of our standard verifications after we've enabled multiseal - step "verify_vault_version" { - description = global.description.verify_vault_version - module = module.vault_verify_version - depends_on = [step.wait_for_seal_rewrap] + step "run_verify_blackbox_tests" { + description = global.description.run_verify_blackbox_tests + module = module.vault_run_blackbox_test + depends_on = [step.wait_for_seal_rewrap, step.get_vault_cluster_ips] providers = { enos = local.enos_provider[matrix.distro] @@ -778,14 +778,47 @@ scenario "seal_ha" { ] variables { - hosts = step.create_vault_cluster_targets.hosts - vault_addr = step.create_vault_cluster.api_addr_localhost + leader_host = step.get_vault_cluster_ips.leader_host + leader_public_ip = step.get_vault_cluster_ips.leader_public_ip + vault_root_token = step.create_vault_cluster.root_token + test_package = "./vault/external_tests/blackbox/verify" + test_names = ["TestVaultServerVersion"] vault_edition = matrix.edition - vault_install_dir = global.vault_install_dir[matrix.artifact_type] vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + + } + } + + step "run_verify_blackbox_tests_remote" { + description = global.description.run_verify_blackbox_tests_remote + module = module.vault_run_blackbox_test + depends_on = [step.run_verify_blackbox_tests] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = [ + quality.vault_version_build_date, + quality.vault_version_edition, + quality.vault_version_release, + ] + + variables { + leader_host = step.get_vault_cluster_ips.leader_host + leader_public_ip = step.get_vault_cluster_ips.leader_public_ip vault_root_token = step.create_vault_cluster.root_token + test_package = "./vault/external_tests/blackbox/verify" + test_names = ["TestVaultCLIVersionLocal"] + vault_edition = matrix.edition + vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version + vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision + vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + } } diff --git a/enos/enos-scenario-smoke.hcl b/enos/enos-scenario-smoke.hcl index 908ae61d95..bb9bc2766f 100644 --- a/enos/enos-scenario-smoke.hcl +++ b/enos/enos-scenario-smoke.hcl @@ -501,10 +501,10 @@ scenario "smoke" { } } - step "verify_vault_version" { - description = global.description.verify_vault_version - module = module.vault_verify_version - depends_on = [step.verify_vault_unsealed] + step "run_verify_blackbox_tests" { + description = global.description.run_verify_blackbox_tests + module = module.vault_run_blackbox_test + depends_on = [step.verify_vault_unsealed, step.get_vault_cluster_ips] providers = { enos = local.enos_provider[matrix.distro] @@ -519,14 +519,47 @@ scenario "smoke" { ] variables { - hosts = step.create_vault_cluster_targets.hosts - vault_addr = step.create_vault_cluster.api_addr_localhost + leader_host = step.get_vault_cluster_ips.leader_host + leader_public_ip = step.get_vault_cluster_ips.leader_public_ip + vault_root_token = step.create_vault_cluster.root_token + test_package = "./vault/external_tests/blackbox/verify" + test_names = ["TestVaultServerVersion"] vault_edition = matrix.edition - vault_install_dir = global.vault_install_dir[matrix.artifact_type] vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + + } + } + + step "run_verify_blackbox_tests_remote" { + description = global.description.run_verify_blackbox_tests_remote + module = module.vault_run_blackbox_test + depends_on = [step.run_verify_blackbox_tests] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = [ + quality.vault_version_build_date, + quality.vault_version_edition, + quality.vault_version_release, + ] + + variables { + leader_host = step.get_vault_cluster_ips.leader_host + leader_public_ip = step.get_vault_cluster_ips.leader_public_ip vault_root_token = step.create_vault_cluster.root_token + test_package = "./vault/external_tests/blackbox/verify" + test_names = ["TestVaultCLIVersionLocal"] + vault_edition = matrix.edition + vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version + vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision + vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + } } @@ -597,7 +630,7 @@ scenario "smoke" { depends_on = [ step.get_vault_cluster_ips, step.vault_remove_node_and_verify, - step.verify_vault_version + step.run_verify_blackbox_tests ] providers = { diff --git a/enos/enos-scenario-upgrade.hcl b/enos/enos-scenario-upgrade.hcl index db1b6953f8..a717b756ca 100644 --- a/enos/enos-scenario-upgrade.hcl +++ b/enos/enos-scenario-upgrade.hcl @@ -695,10 +695,10 @@ scenario "upgrade" { } } - step "verify_vault_version" { - description = global.description.verify_vault_version - module = module.vault_verify_version - depends_on = [step.verify_vault_unsealed] + step "run_verify_blackbox_tests" { + description = global.description.run_verify_blackbox_tests + module = module.vault_run_blackbox_test + depends_on = [step.verify_vault_unsealed, step.get_vault_cluster_ips] providers = { enos = local.enos_provider[matrix.distro] @@ -713,14 +713,47 @@ scenario "upgrade" { ] variables { - hosts = step.create_vault_cluster_targets.hosts - vault_addr = step.create_vault_cluster.api_addr_localhost + leader_host = step.get_vault_cluster_ips.leader_host + leader_public_ip = step.get_vault_cluster_ips.leader_public_ip + vault_root_token = step.create_vault_cluster.root_token + test_package = "./vault/external_tests/blackbox/verify" + test_names = ["TestVaultServerVersion"] vault_edition = matrix.edition - vault_install_dir = global.vault_install_dir[matrix.artifact_type] vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + + } + } + + step "run_verify_blackbox_tests_remote" { + description = global.description.run_verify_blackbox_tests_remote + module = module.vault_run_blackbox_test + depends_on = [step.run_verify_blackbox_tests] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = [ + quality.vault_version_build_date, + quality.vault_version_edition, + quality.vault_version_release, + ] + + variables { + leader_host = step.get_vault_cluster_ips.leader_host + leader_public_ip = step.get_vault_cluster_ips.leader_public_ip vault_root_token = step.create_vault_cluster.root_token + test_package = "./vault/external_tests/blackbox/verify" + test_names = ["TestVaultCLIVersionLocal"] + vault_edition = matrix.edition + vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version + vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision + vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + } } diff --git a/enos/modules/vault_run_blackbox_test/main.tf b/enos/modules/vault_run_blackbox_test/main.tf index bb31e24530..700549f0f5 100644 --- a/enos/modules/vault_run_blackbox_test/main.tf +++ b/enos/modules/vault_run_blackbox_test/main.tf @@ -32,19 +32,27 @@ resource "random_string" "test_id" { } resource "enos_local_exec" "run_blackbox_test" { - scripts = [abspath("${path.module}/scripts/run-test.sh")] - environment = merge({ - VAULT_TOKEN = var.vault_root_token - VAULT_ADDR = var.vault_addr != null ? var.vault_addr : "http://${var.leader_public_ip}:8200" - VAULT_TEST_PACKAGE = var.test_package - VAULT_TEST_MATRIX = length(local.test_names) > 0 ? local_file.test_matrix.filename : "" - VAULT_EDITION = var.vault_edition - # PATH and Go-related environment variables are inherited from the calling process - }, var.vault_namespace != null ? { - VAULT_NAMESPACE = var.vault_namespace - } : {}, local.ldap_environment, local.postgres_environment, local.mongodb_environment - ) + scripts = [abspath("${path.module}/scripts/run-test.sh")] depends_on = [local_file.test_matrix] + + environment = merge( + { + VAULT_TOKEN = var.vault_root_token + VAULT_ADDR = var.vault_addr != null ? var.vault_addr : "http://${var.leader_public_ip}:8200" + VAULT_TEST_PACKAGE = var.test_package + VAULT_TEST_MATRIX = length(local.test_names) > 0 ? local_file.test_matrix.filename : "" + VAULT_EDITION = var.vault_edition + # PATH and Go-related environment variables are inherited from the calling process + }, + var.vault_namespace != null ? { VAULT_NAMESPACE = var.vault_namespace } : {}, + var.vault_product_version != null ? { VAULT_VERSION = var.vault_product_version } : {}, + var.vault_revision != null ? { VAULT_REVISION = var.vault_revision } : {}, + var.vault_build_date != null ? { VAULT_BUILD_DATE = var.vault_build_date } : {}, + var.vault_install_dir != null ? { VAULT_INSTALL_DIR = var.vault_install_dir } : {}, + local.ldap_environment, + local.postgres_environment, + local.mongodb_environment + ) } # Local variables for LDAP environment setup diff --git a/enos/modules/vault_run_blackbox_test/outputs.tf b/enos/modules/vault_run_blackbox_test/outputs.tf index c22f750712..7ce0860487 100644 --- a/enos/modules/vault_run_blackbox_test/outputs.tf +++ b/enos/modules/vault_run_blackbox_test/outputs.tf @@ -6,7 +6,6 @@ output "test_result" { value = enos_local_exec.run_blackbox_test.stdout } - output "test_results_summary" { description = "Summary of test results for dashboards" value = { diff --git a/enos/modules/vault_run_blackbox_test/scripts/run-test.sh b/enos/modules/vault_run_blackbox_test/scripts/run-test.sh index e4c44b0451..a2f0675d58 100755 --- a/enos/modules/vault_run_blackbox_test/scripts/run-test.sh +++ b/enos/modules/vault_run_blackbox_test/scripts/run-test.sh @@ -84,15 +84,15 @@ if [[ "${VAULT_ADDR}" == http://* ]]; then fi case $VAULT_EDITION in - ent | ent.hsm | ent.hsm.fips1402 | ent.hsm.fips1403 | ent.fips1403 | ent.fips1402) - tags="-tags=ent,enterprise" - ;; - ce) - tags="" - ;; - *) - fail "unknown VAULT_EDITION: $VAULT_EDITION" - ;; + ent | ent.hsm | ent.hsm.fips1402 | ent.hsm.fips1403 | ent.fips1403 | ent.fips1402) + tags="-tags=ent,enterprise" + ;; + ce) + tags="" + ;; + *) + fail "unknown VAULT_EDITION: $VAULT_EDITION" + ;; esac # Build gotestsum command based on whether we have specific tests diff --git a/enos/modules/vault_run_blackbox_test/variables.tf b/enos/modules/vault_run_blackbox_test/variables.tf index 032d1ab391..62814e18b7 100644 --- a/enos/modules/vault_run_blackbox_test/variables.tf +++ b/enos/modules/vault_run_blackbox_test/variables.tf @@ -53,3 +53,27 @@ variable "vault_edition" { description = "The Vault edition (ce, ent, ent.hsm, ent.fips1402, ent.hsm.fips1402)" default = "ent" } + +variable "vault_product_version" { + type = string + description = "The Vault product version (e.g., 1.15.0)" + default = null +} + +variable "vault_revision" { + type = string + description = "The Vault git revision/commit SHA" + default = null +} + +variable "vault_build_date" { + type = string + description = "The Vault build date" + default = null +} + +variable "vault_install_dir" { + type = string + description = "The directory where Vault is installed" + default = null +} diff --git a/enos/modules/vault_verify_version/main.tf b/enos/modules/vault_verify_version/main.tf deleted file mode 100644 index 4114945a9f..0000000000 --- a/enos/modules/vault_verify_version/main.tf +++ /dev/null @@ -1,100 +0,0 @@ -# Copyright IBM Corp. 2016, 2025 -# SPDX-License-Identifier: BUSL-1.1 - -terraform { - required_providers { - enos = { - source = "registry.terraform.io/hashicorp-forge/enos" - } - } -} - -variable "hosts" { - type = map(object({ - ipv6 = string - private_ip = string - public_ip = string - })) - description = "The Vault cluster instances that were created" -} - -variable "vault_addr" { - type = string - description = "The local vault API listen address" -} - -variable "vault_build_date" { - type = string - description = "The Vault artifact build date" - default = null -} - -variable "vault_edition" { - type = string - description = "The Vault product edition" - default = null -} - -variable "vault_install_dir" { - type = string - description = "The directory where the Vault binary will be installed" -} - -variable "vault_product_version" { - type = string - description = "The Vault product version" - default = null -} - -variable "vault_revision" { - type = string - description = "The Vault product revision" - default = null -} - -variable "vault_root_token" { - type = string - description = "The Vault root token" - default = null -} - -resource "enos_remote_exec" "verify_cli_version" { - for_each = var.hosts - - environment = { - VAULT_ADDR = var.vault_addr, - VAULT_BUILD_DATE = var.vault_build_date, - VAULT_EDITION = var.vault_edition, - VAULT_INSTALL_DIR = var.vault_install_dir, - VAULT_REVISION = var.vault_revision, - VAULT_TOKEN = var.vault_root_token, - VAULT_VERSION = var.vault_product_version, - } - - scripts = [abspath("${path.module}/scripts/verify-cli-version.sh")] - - transport = { - ssh = { - host = each.value.public_ip - } - } -} - -resource "enos_remote_exec" "verify_cluster_version" { - for_each = var.hosts - - environment = { - VAULT_ADDR = var.vault_addr, - VAULT_BUILD_DATE = var.vault_build_date, - VAULT_TOKEN = var.vault_root_token, - VAULT_VERSION = var.vault_product_version, - } - - scripts = [abspath("${path.module}/scripts/verify-cluster-version.sh")] - - transport = { - ssh = { - host = each.value.public_ip - } - } -} diff --git a/enos/modules/vault_verify_version/scripts/verify-cli-version.sh b/enos/modules/vault_verify_version/scripts/verify-cli-version.sh deleted file mode 100644 index 3834b3bbf7..0000000000 --- a/enos/modules/vault_verify_version/scripts/verify-cli-version.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash -# Copyright IBM Corp. 2016, 2025 -# SPDX-License-Identifier: BUSL-1.1 - -# Verify the Vault "version" includes the correct base version, build date, -# revision SHA, and edition metadata. -set -e - -fail() { - echo "$1" 1>&2 - exit 1 -} - -[[ -z "$VAULT_ADDR" ]] && fail "VAULT_ADDR env variable has not been set" -[[ -z "$VAULT_BUILD_DATE" ]] && fail "VAULT_BUILD_DATE env variable has not been set" -[[ -z "$VAULT_EDITION" ]] && fail "VAULT_EDITION env variable has not been set" -[[ -z "$VAULT_INSTALL_DIR" ]] && fail "VAULT_INSTALL_DIR env variable has not been set" -[[ -z "$VAULT_REVISION" ]] && fail "VAULT_REVISION env variable has not been set" -[[ -z "$VAULT_TOKEN" ]] && fail "VAULT_TOKEN env variable has not been set" -[[ -z "$VAULT_VERSION" ]] && fail "VAULT_VERSION env variable has not been set" - -binpath=${VAULT_INSTALL_DIR}/vault -edition=${VAULT_EDITION} -version=${VAULT_VERSION} -sha=${VAULT_REVISION} -build_date=${VAULT_BUILD_DATE} - -test -x "$binpath" || fail "unable to locate vault binary at $binpath" -version_expected="Vault v$version ($sha), built $build_date" - -case "$edition" in - *ce) ;; - *ent) ;; - *ent.hsm) version_expected="$version_expected (cgo)" ;; - *ent.fips1403) version_expected="$version_expected (cgo)" ;; - *ent.hsm.fips1403) version_expected="$version_expected (cgo)" ;; - *) fail "Unknown Vault edition: ($edition)" ;; -esac - -version_expected_nosha=$(echo "$version_expected" | awk '!($3="")' | sed 's/ / /' | sed -e 's/[[:space:]]*$//') -version_output=$("$binpath" version) - -if [[ "$version_output" == "$version_expected_nosha" ]] || [[ "$version_output" == "$version_expected" ]]; then - echo "Version verification succeeded!" -else - msg="$(printf "\nThe Vault cluster did not match the expected version, expected:\n%s\nor\n%s\ngot:\n%s" "$version_expected" "$version_expected_nosha" "$version_output")" - if type diff &> /dev/null; then - # Diff exits non-zero if we have a diff, which we want, so we'll guard against failing early. - if ! version_diff=$(diff <(echo "$version_expected") <(echo "$version_output") -u -L expected -L got); then - msg="$(printf "\nThe Vault cluster did not match the expected version:\n%s" "$version_diff")" - fi - fi - - fail "$msg" -fi diff --git a/enos/modules/vault_verify_version/scripts/verify-cluster-version.sh b/enos/modules/vault_verify_version/scripts/verify-cluster-version.sh deleted file mode 100644 index 1f131c263a..0000000000 --- a/enos/modules/vault_verify_version/scripts/verify-cluster-version.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash -# Copyright IBM Corp. 2016, 2025 -# SPDX-License-Identifier: BUSL-1.1 - -# Verify the Vault "version" includes the correct base version, build date, -# revision SHA, and edition metadata. -set -e - -fail() { - echo "$1" 1>&2 - exit 1 -} - -[[ -z "$VAULT_ADDR" ]] && fail "VAULT_ADDR env variable has not been set" -[[ -z "$VAULT_BUILD_DATE" ]] && fail "VAULT_BUILD_DATE env variable has not been set" -[[ -z "$VAULT_TOKEN" ]] && fail "VAULT_TOKEN env variable has not been set" -[[ -z "$VAULT_VERSION" ]] && fail "VAULT_VERSION env variable has not been set" - -# The sys/version-history endpoint only includes major.minor.patch, any other semver fields need to -# be stripped out. -if ! version=$(cut -d + -f1 <<< "$VAULT_VERSION" | cut -d - -f1); then - fail "failed to parse the expected version: $version" -fi - -if ! vh=$(curl -s -X LIST -H "X-Vault-Token: $VAULT_TOKEN" http://127.0.0.1:8200/v1/sys/version-history | jq -eMc '.data'); then - fail "failed to Vault cluster version history: $vh" -fi - -if ! out=$(jq -eMc --arg version "$version" '.keys | contains([$version])' <<< "$vh"); then - fail "cluster version history does not include our expected version: expected: $version, versions: $(jq -eMc '.keys' <<< "$vh"): output: $out" -fi - -if ! out=$(jq -eMc --arg version "$version" --arg bd "$VAULT_BUILD_DATE" '.key_info[$version].build_date == $bd' <<< "$vh"); then - fail "cluster version history build date is not the expected date: expected: true, expected date: $VAULT_BUILD_DATE, key_info: $(jq -eMc '.key_info' <<< "$vh"), output: $out" -fi - -printf "Cluster version information is valid!: %s\n" "$vh" diff --git a/sdk/helper/testcluster/blackbox/session_status.go b/sdk/helper/testcluster/blackbox/session_status.go index e158866653..3b9f97ad63 100644 --- a/sdk/helper/testcluster/blackbox/session_status.go +++ b/sdk/helper/testcluster/blackbox/session_status.go @@ -42,6 +42,87 @@ func (s *Session) AssertUnsealedAny() { s.t.Logf("Vault is unsealed (seal type: %s)", status.Type) } +// AssertVersion verifies the Vault version via sys/version-history API +func (s *Session) AssertVersion(version string) { + s.t.Helper() + + // strip off any version metadata + b, _, _ := strings.Cut(version, "+") + expectedVersion, _, _ := strings.Cut(b, "-") + + secret, err := s.Client.Logical().List("sys/version-history") + require.NoError(s.t, err) + + keysRaw, ok := secret.Data["keys"].([]any) + if !ok { + s.t.Fatal("sys/version-history missing 'keys'") + } + + found := false + for _, k := range keysRaw { + if kStr, ok := k.(string); ok && kStr == expectedVersion { + found = true + break + } + } + + if !found { + s.t.Fatalf("expected to find %s in version history but didn't", expectedVersion) + } +} + +// AssertBuildDate verifies the Vault build date via sys/version-history API +func (s *Session) AssertBuildDate(version, buildDate string) { + s.t.Helper() + + // strip off any version metadata + b, _, _ := strings.Cut(version, "+") + expectedVersion, _, _ := strings.Cut(b, "-") + + secret, err := s.Client.Logical().List("sys/version-history") + require.NoError(s.t, err) + + keyInfoRaw, ok := secret.Data["key_info"].(map[string]any) + if !ok { + s.t.Fatal("sys/version-history missing 'key_info'") + } + + versionInfo, ok := keyInfoRaw[expectedVersion].(map[string]any) + if !ok { + s.t.Fatalf("version %s not found in key_info", expectedVersion) + } + + actualBuildDate, ok := versionInfo["build_date"].(string) + if !ok { + s.t.Fatal("build_date not found in version info") + } + + if actualBuildDate != buildDate { + s.t.Fatalf("build date mismatch: expected %s, got %s", buildDate, actualBuildDate) + } +} + +// AssertRevision verifies the Vault revision/SHA from CLI output +func (s *Session) AssertRevision(revision string) { + s.t.Helper() + + // make sure the binary exists first + _, err := exec.LookPath("vault") + require.NoError(s.t, err) + + cmd := exec.Command("vault", "version") + out, err := cmd.CombinedOutput() + require.NoError(s.t, err) + + output := string(out) + + if !strings.Contains(output, fmt.Sprintf("'%s'", revision)) { + s.t.Fatalf("CLI revision mismatch. expected %s in output: %s", revision, output) + } +} + +// AssertCLIVersion verifies the complete Vault CLI version output +// Deprecated: Use AssertVersion, AssertBuildDate, and AssertRevision separately for more granular testing func (s *Session) AssertCLIVersion(version, sha, buildDate, edition string) { s.t.Helper() @@ -70,32 +151,12 @@ func (s *Session) AssertCLIVersion(version, sha, buildDate, edition string) { } } -func (s *Session) AssertServerVersion(version string) { +// AssertServerVersion verifies the Vault server version and build date via sys/version-history API +// Deprecated: Use AssertVersion and AssertBuildDate separately for more granular testing +func (s *Session) AssertServerVersion(version, buildDate string) { s.t.Helper() - - // strip off any version metadata - b, _, _ := strings.Cut(version, "+") - expectedVersion, _, _ := strings.Cut(b, "-") - - secret, err := s.Client.Logical().List("sys/version-history") - require.NoError(s.t, err) - - keysRaw, ok := secret.Data["keys"].([]any) - if !ok { - s.t.Fatal("sys/version-history missing 'keys'") - } - - found := false - for _, k := range keysRaw { - if kStr, ok := k.(string); ok && kStr == expectedVersion { - found = true - break - } - } - - if !found { - s.t.Fatalf("expected to find %s in version history but didn't", expectedVersion) - } + s.AssertVersion(version) + s.AssertBuildDate(version, buildDate) } func (s *Session) AssertReplicationDisabled() { diff --git a/vault/external_tests/blackbox/verify/version_verification_test.go b/vault/external_tests/blackbox/verify/version_verification_test.go new file mode 100644 index 0000000000..61e0ee0dd6 --- /dev/null +++ b/vault/external_tests/blackbox/verify/version_verification_test.go @@ -0,0 +1,31 @@ +// Copyright IBM Corp. 2025, 2026 +// SPDX-License-Identifier: BUSL-1.1 + +package verify + +import ( + "os" + "testing" + + "github.com/hashicorp/vault/sdk/helper/testcluster/blackbox" +) + +// TestVaultServerVersion verifies the Vault server version via sys/version-history API +// This test runs from CI/GitHub runners and connects to the Vault cluster via API +func TestVaultServerVersion(t *testing.T) { + t.Parallel() + + version := os.Getenv("VAULT_VERSION") + if version == "" { + t.Fatal("VAULT_VERSION environment variable is required") + } + + buildDate := os.Getenv("VAULT_BUILD_DATE") + if buildDate == "" { + t.Fatal("VAULT_BUILD_DATE environment variable is required") + } + + v := blackbox.New(t) + v.AssertVersion(version) + v.AssertBuildDate(version, buildDate) +}