From 40a70edc0350b7b9295e270fde183aa15550f445 Mon Sep 17 00:00:00 2001 From: Vault Automation Date: Thu, 6 Nov 2025 13:59:40 -0500 Subject: [PATCH] Add docker based backend to the cloud scenario (#9751) (#10626) * Add docker based backed * new line * Add validation * Add cloud_docker_vault_cluster * Unify cloud scenario outputs * Use min_vault_version consistently across both modules * random network name for docker * Add local build for docker * Use environment instead of backend * make use of existing modules for docker and k8s * connect the peers * formatting * copyright * Remove old duplicated code * use enos local exec * get version locally * Dont use local time * adjust bin path for docker * use root dockerfile * get dockerfile to work * Build docker image from correct binary location * Fix it... maybe * Add docker admin token * whitespace * formatting and comment cleanup * formatting * undo * Apply suggestion from @ryancragun * Move build to make * Default to local * Revert k8s changes * Add admint token * Clean map * whitespace * whitespace * Pull out k8 changes and vault_cluster_raft * Some cleaning changes * whitespace * Naming cleanup --------- Co-authored-by: Luis (LT) Carbonell Co-authored-by: Ryan Cragun --- Makefile | 4 + enos/enos-modules.hcl | 12 + enos/enos-providers.hcl | 3 + enos/enos-terraform.hcl | 4 + enos/modules/build_local/main.tf | 40 +- enos/modules/build_local/scripts/build.sh | 23 +- .../cloud_docker_vault_cluster/main.tf | 397 ++++++++++++++++++ enos/modules/docker_namespace_token/main.tf | 125 ++++++ enos/modules/docker_network/main.tf | 31 ++ enos/modules/hcp/create_admin_token/main.tf | 14 + enos/modules/hcp/create_vault_cluster/main.tf | 38 ++ scripts/ci-helper.sh | 393 +++++++++-------- 12 files changed, 898 insertions(+), 186 deletions(-) create mode 100644 enos/modules/cloud_docker_vault_cluster/main.tf create mode 100644 enos/modules/docker_namespace_token/main.tf create mode 100644 enos/modules/docker_network/main.tf diff --git a/Makefile b/Makefile index ecda8de311..02ff62cf26 100644 --- a/Makefile +++ b/Makefile @@ -366,6 +366,10 @@ ci-build-ui: ci-bundle: @$(CURDIR)/scripts/ci-helper.sh bundle +.PHONY: ci-copy-binary +ci-copy-binary: + @$(CURDIR)/scripts/ci-helper.sh copy-binary + .PHONY: ci-get-artifact-basename ci-get-artifact-basename: @$(CURDIR)/scripts/ci-helper.sh artifact-basename diff --git a/enos/enos-modules.hcl b/enos/enos-modules.hcl index 9085384506..dee5203073 100644 --- a/enos/enos-modules.hcl +++ b/enos/enos-modules.hcl @@ -56,6 +56,18 @@ module "create_vpc" { common_tags = var.tags } +module "cloud_docker_vault_cluster" { + source = "./modules/cloud_docker_vault_cluster" +} + +module "docker_namespace_token" { + source = "./modules/docker_namespace_token" +} + +module "docker_network" { + source = "./modules/docker_network" +} + module "choose_follower_host" { source = "./modules/choose_follower_host" } diff --git a/enos/enos-providers.hcl b/enos/enos-providers.hcl index fe4d3e53d6..a9bddf33bb 100644 --- a/enos/enos-providers.hcl +++ b/enos/enos-providers.hcl @@ -27,3 +27,6 @@ provider "enos" "ubuntu" { provider "hcp" "default" { } + +provider "docker" "default" { +} diff --git a/enos/enos-terraform.hcl b/enos/enos-terraform.hcl index 41e714ca1a..93526556ef 100644 --- a/enos/enos-terraform.hcl +++ b/enos/enos-terraform.hcl @@ -24,6 +24,10 @@ terraform "default" { source = "hashicorp/aws" } + docker = { + source = "kreuzwerker/docker" + } + enos = { source = "registry.terraform.io/hashicorp-forge/enos" version = ">= 0.4.0" diff --git a/enos/modules/build_local/main.tf b/enos/modules/build_local/main.tf index 106157f8a5..6f32e8efc2 100644 --- a/enos/modules/build_local/main.tf +++ b/enos/modules/build_local/main.tf @@ -1,6 +1,28 @@ # Copyright IBM Corp. 2016, 2025 # SPDX-License-Identifier: BUSL-1.1 +# Vault Local Build Module +# +# This module builds Vault binaries locally and produces artifacts for different use cases. +# The module enforces that at least one output artifact is created to prevent silent no-op execution. +# +# Supported workflows: +# 1. ZIP Bundle Only: artifact_path specified, docker_bin_path null +# - Builds binary to dist/ +# - Creates zip bundle at artifact_path +# +# 2. Docker Only: docker_bin_path specified, artifact_path null +# - Builds binary to dist/ +# - Copies binary to docker_bin_path for Docker image creation +# - No zip bundle created +# +# 3. Both ZIP and Docker: Both artifact_path and docker_bin_path specified +# - Builds binary to dist/ +# - Creates zip bundle at artifact_path +# - Copies binary to docker_bin_path for Docker image creation +# +# The validation ensures at least one of artifact_path or docker_bin_path is specified. + terraform { required_providers { enos = { @@ -10,7 +32,9 @@ terraform { } variable "artifact_path" { - description = "Where to create the zip bundle of the Vault build" + type = string + description = "Where to create the zip bundle of the Vault build. If null, no zip bundle will be created." + default = null } variable "build_tags" { @@ -36,6 +60,17 @@ variable "goos" { default = "linux" } +variable "docker_bin_path" { + type = string + description = "Path to copy the built binary for Docker image creation. When specified, the binary is copied to this location for subsequent Docker image builds. If null, no Docker-specific binary copy is performed." + default = null + + validation { + condition = var.artifact_path != null || var.docker_bin_path != null + error_message = "At least one of 'artifact_path' (for zip bundle) or 'docker_bin_path' (for Docker builds) must be specified. The module must produce at least one output artifact." + } +} + variable "artifactory_host" { default = null } variable "artifactory_repo" { default = null } variable "artifactory_token" { default = null } @@ -58,11 +93,12 @@ resource "enos_local_exec" "build" { BASE_VERSION = module.local_metadata.version_base BIN_PATH = abspath("${path.module}/../../../dist") BUILD_UI = tostring(var.build_ui) - BUNDLE_PATH = abspath(var.artifact_path) + BUNDLE_PATH = var.artifact_path != null ? abspath(var.artifact_path) : "" GO_TAGS = join(" ", var.build_tags) GOARCH = var.goarch GOOS = var.goos PRERELEASE_VERSION = module.local_metadata.version_pre VERSION_METADATA = module.local_metadata.version_meta + CUSTOM_BIN_PATH = var.docker_bin_path != null ? abspath("${path.module}/../../../${var.docker_bin_path}") : "" } } diff --git a/enos/modules/build_local/scripts/build.sh b/enos/modules/build_local/scripts/build.sh index ee6b889ff4..66b1dca808 100755 --- a/enos/modules/build_local/scripts/build.sh +++ b/enos/modules/build_local/scripts/build.sh @@ -2,6 +2,18 @@ # Copyright IBM Corp. 2016, 2025 # SPDX-License-Identifier: BUSL-1.1 +# This script builds Vault binaries and optionally packages them. +# +# Two distinct workflows are supported: +# 1. Standard build: Builds to dist/ and creates zip bundle from dist/ +# 2. Target path build: Builds to dist/, copies to TARGET_BIN_PATH, skips bundling +# (bundling is skipped to avoid confusion when binary exists in multiple locations) +# +# Environment variables: +# - BUILD_UI: Set to "true" to build UI components +# - TARGET_BIN_PATH: If set, copies built binary to this location instead of bundling +# - BUNDLE_PATH: If set (and TARGET_BIN_PATH is not), creates zip bundle at this path + set -eux -o pipefail # Install yarn so we can build the UI @@ -20,5 +32,12 @@ make ci-build popd > /dev/null -echo "--> Bundling $BIN_PATH/* to $BUNDLE_PATH" -zip -r -j "$BUNDLE_PATH" "$BIN_PATH/" +if [ -n "$TARGET_BIN_PATH" ]; then + echo "--> Target binary path specified, copying binary and skipping bundle" + make -C "$root_dir" ci-copy-binary +elif [ -n "$BUNDLE_PATH" ]; then + echo "--> Creating zip bundle from dist/" + make -C "$root_dir" ci-bundle +else + echo "--> No post-build packaging requested (neither TARGET_BIN_PATH nor BUNDLE_PATH specified)" +fi diff --git a/enos/modules/cloud_docker_vault_cluster/main.tf b/enos/modules/cloud_docker_vault_cluster/main.tf new file mode 100644 index 0000000000..8248e2c773 --- /dev/null +++ b/enos/modules/cloud_docker_vault_cluster/main.tf @@ -0,0 +1,397 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +terraform { + required_providers { + docker = { + source = "kreuzwerker/docker" + version = "~> 3.0" + } + enos = { + source = "registry.terraform.io/hashicorp-forge/enos" + } + } +} + +variable "min_vault_version" { + type = string + description = "The minimum Vault version to deploy (e.g., 1.15.0 or v1.15.0+ent)" +} + +variable "vault_edition" { + type = string + description = "The edition of Vault to deploy (ent, ce, ent.fips1403)" + default = "ent" + + validation { + condition = contains(["ent", "ce", "ent.fips1403"], var.vault_edition) + error_message = "vault_edition must be one of: ent, ce, ent.fips1403" + } +} + +variable "vault_license" { + type = string + description = "The Vault Enterprise license" + default = null + sensitive = true +} + +variable "cluster_name" { + type = string + description = "The name of the Vault cluster" + default = "vault" +} + +variable "container_count" { + type = number + description = "Number of Vault containers to create" + default = 3 +} + +variable "vault_port" { + type = number + description = "The port Vault listens on" + default = 8200 +} + +variable "use_local_build" { + type = bool + description = "If true, build a local Docker image from the current branch instead of pulling from Docker Hub" + default = false +} + +# HCP-specific variables (ignored but accepted for compatibility) +variable "network_name" { + type = string + description = "Ignored - for HCP compatibility only" + default = "" +} + +variable "tier" { + type = string + description = "Ignored - for HCP compatibility only" + default = "" +} + +# Generate a random suffix for the network name to avoid conflicts +resource "random_string" "network_suffix" { + length = 8 + lower = true + upper = false + numeric = true + special = false +} + +# Create Docker network +resource "docker_network" "cluster" { + name = "${var.cluster_name}-network-${random_string.network_suffix.result}" +} + +locals { + # Parse min_vault_version to extract the version number + # e.g., "v1.15.0+ent" -> "1.15.0" or "v1.15.0+ent-2cf0b2f" -> "1.15.0" + vault_version = trimprefix(split("+", var.min_vault_version)[0], "v") + + image_map = { + "ent" = "hashicorp/vault-enterprise" + "ce" = "hashicorp/vault" + "ent.fips1403" = "hashicorp/vault-enterprise-fips" + } + target_map = { + "ent" = "ubi" + "ce" = "ubi" + "ent.fips1403" = "ubi-fips" + } + image = local.image_map[var.vault_edition] + tag_suffix = var.vault_edition == "ce" ? "" : "-ent" + image_tag = "${local.vault_version}${local.tag_suffix}" + local_tag = "vault-local-${var.vault_edition}:${local.vault_version}" + dockerfile = "Dockerfile" + target = local.target_map[var.vault_edition] +} + +# Pull image from Docker Hub (when not using local build) +resource "docker_image" "vault_remote" { + count = var.use_local_build ? 0 : 1 + name = "${local.image}:${local.image_tag}" +} + +# Build image from local Dockerfile (when using local build) +resource "docker_image" "vault_local" { + count = var.use_local_build ? 1 : 0 + name = local.local_tag + keep_locally = true + + build { + context = "${path.module}/../../.." + dockerfile = local.dockerfile + target = local.target + tag = [local.local_tag] + pull_parent = true + build_args = { + BIN_NAME = "vault" + TARGETOS = "linux" + TARGETARCH = "amd64" + NAME = "vault" + PRODUCT_VERSION = local.vault_version + PRODUCT_REVISION = "local" + LICENSE_SOURCE = "LICENSE" + LICENSE_DEST = "/usr/share/doc/vault" + } + } + +} + +locals { + # Generate Vault configuration for each node + vault_config_template = <<-EOF + ui = true + listener "tcp" { + address = "0.0.0.0:${var.vault_port}" + cluster_address = "0.0.0.0:8201" + tls_disable = true + } + + storage "raft" { + path = "/vault/data" + node_id = "node%s" + } + + disable_mlock = true + EOF +} + +# Using tmpfs for Raft data (in-memory, no persistence needed for testing) + +resource "docker_container" "vault" { + count = var.container_count + name = "${var.cluster_name}-${count.index}" + image = var.use_local_build ? docker_image.vault_local[0].name : docker_image.vault_remote[0].image_id + + networks_advanced { + name = docker_network.cluster.name + } + + ports { + internal = var.vault_port + external = var.vault_port + count.index + } + + tmpfs = { + "/vault/data" = "rw,noexec,nosuid,size=100m" + } + + upload { + content = format(local.vault_config_template, count.index) + file = "/vault/config/vault.hcl" + } + + + user = "root" + + env = concat( + [ + "VAULT_API_ADDR=http://${var.cluster_name}-${count.index}:${var.vault_port}", + "VAULT_CLUSTER_ADDR=http://${var.cluster_name}-${count.index}:8201", + "SKIP_SETCAP=true", + "SKIP_CHOWN=true", + ], + var.vault_license != null ? ["VAULT_LICENSE=${var.vault_license}"] : [] + ) + + capabilities { + add = ["IPC_LOCK"] + } + + command = ["vault", "server", "-config=/vault/config/vault.hcl"] + + restart = "no" +} + +locals { + instance_indexes = [for idx in range(var.container_count) : tostring(idx)] + leader_idx = 0 + followers_idx = range(1, var.container_count) + + vault_address = "http://127.0.0.1:${var.vault_port}" + leader_api_addr = "http://${var.cluster_name}-${local.leader_idx}:${var.vault_port}" +} + +# Initialize Vault on the leader +resource "enos_local_exec" "init_leader" { + inline = [ + <<-EOT + # Wait for Vault to be ready (output to stderr to keep stdout clean) + for i in 1 2 3 4 5 6 7 8 9 10; do + if docker exec -e VAULT_ADDR=http://127.0.0.1:${var.vault_port} ${docker_container.vault[local.leader_idx].name} vault status 2>&1 | grep -q "Initialized.*false"; then + break + fi + echo "Waiting for Vault to start (attempt $i/10)..." >&2 + sleep 2 + done + + # Initialize Vault and output JSON to stdout + docker exec -e VAULT_ADDR=http://127.0.0.1:${var.vault_port} ${docker_container.vault[local.leader_idx].name} vault operator init \ + -key-shares=1 \ + -key-threshold=1 \ + -format=json + EOT + ] + + depends_on = [docker_container.vault] +} + +locals { + init_data = jsondecode(enos_local_exec.init_leader.stdout) + unseal_key = local.init_data.unseal_keys_b64[0] + root_token = local.init_data.root_token +} + +# Unseal the leader +resource "enos_local_exec" "unseal_leader" { + inline = [ + "docker exec -e VAULT_ADDR=http://127.0.0.1:${var.vault_port} ${docker_container.vault[local.leader_idx].name} vault operator unseal ${local.unseal_key}" + ] + + depends_on = [enos_local_exec.init_leader] +} + +# Join followers to Raft cluster and unseal them +resource "enos_local_exec" "join_followers" { + count = length(local.followers_idx) + + inline = [ + <<-EOT + # Wait for Vault to be ready + for i in 1 2 3 4 5; do + docker exec -e VAULT_ADDR=http://127.0.0.1:${var.vault_port} ${docker_container.vault[local.followers_idx[count.index]].name} vault status > /dev/null 2>&1 && break || sleep 5 + done + + # Join the Raft cluster + docker exec -e VAULT_ADDR=http://127.0.0.1:${var.vault_port} ${docker_container.vault[local.followers_idx[count.index]].name} \ + vault operator raft join ${local.leader_api_addr} + + # Unseal the follower + docker exec -e VAULT_ADDR=http://127.0.0.1:${var.vault_port} ${docker_container.vault[local.followers_idx[count.index]].name} \ + vault operator unseal ${local.unseal_key} + EOT + ] + + depends_on = [enos_local_exec.unseal_leader] +} + +# Outputs that match HCP module interface +output "cloud_provider" { + value = "docker" + description = "The cloud provider (docker for local)" +} + +output "cluster_id" { + value = var.cluster_name + description = "The cluster identifier" +} + +output "created_at" { + value = timestamp() + description = "Timestamp of cluster creation" +} + +output "id" { + value = var.cluster_name + description = "The cluster identifier" +} + +output "namespace" { + value = "root" + description = "The Vault namespace" +} + +output "organization_id" { + value = "docker-local" + description = "The organization identifier" +} + +output "region" { + value = "local" + description = "The region or location" +} + +output "self_link" { + value = "" + description = "Self link to the cluster" +} + +output "state" { + value = "RUNNING" + description = "The state of the cluster" +} + +output "vault_private_endpoint_url" { + value = "" + description = "Private endpoint URL (not applicable for Docker)" +} + +output "vault_proxy_endpoint_url" { + value = "" + description = "Proxy endpoint URL (not applicable for Docker)" +} + +output "vault_public_endpoint_url" { + value = "http://localhost:${var.vault_port}" + description = "Public endpoint URL" +} + +output "vault_version" { + value = local.vault_version + description = "The version of Vault deployed" +} + +# Docker-specific outputs +output "container_names" { + value = docker_container.vault[*].name + description = "The names of the Vault containers" +} + +output "container_ids" { + value = docker_container.vault[*].id + description = "The IDs of the Vault containers" +} + +output "vault_addresses" { + value = [ + for i in range(var.container_count) : + "http://localhost:${var.vault_port + i}" + ] + description = "The addresses of the Vault containers" +} + +output "primary_address" { + value = "http://localhost:${var.vault_port}" + description = "The address of the primary Vault container" +} + +output "network_id" { + value = docker_network.cluster.id + description = "The ID of the created Docker network" +} + +output "network_name" { + value = docker_network.cluster.name + description = "The name of the created Docker network" +} + +output "image_name" { + value = var.use_local_build ? (length(docker_image.vault_local) > 0 ? docker_image.vault_local[0].name : "none") : (length(docker_image.vault_remote) > 0 ? docker_image.vault_remote[0].name : "none") + description = "The Docker image being used" +} + +output "is_local_build" { + value = var.use_local_build + description = "Whether this is using a local build" +} + +output "vault_root_token" { + value = local.root_token + sensitive = true + description = "The root token for the Vault cluster" +} diff --git a/enos/modules/docker_namespace_token/main.tf b/enos/modules/docker_namespace_token/main.tf new file mode 100644 index 0000000000..f7ab739311 --- /dev/null +++ b/enos/modules/docker_namespace_token/main.tf @@ -0,0 +1,125 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +variable "vault_root_token" { + description = "The root token from the Docker Vault cluster" + type = string + sensitive = true + default = null +} + +variable "vault_address" { + description = "The address of the Vault cluster" + type = string + default = null +} + +# HCP compatibility variables (accepted but ignored) +variable "cluster_id" { + description = "Ignored - for HCP compatibility only" + type = string + default = null +} + +terraform { + required_providers { + enos = { + source = "registry.terraform.io/hashicorp-forge/enos" + } + } +} + +# We need the container name to exec into +variable "container_name" { + description = "The name of the Docker container running Vault (Docker only)" + type = string + default = null +} + +variable "namespace_name" { + description = "The name of the namespace to create and generate the token in" + type = string + default = "admin" +} + +# Create namespace using the root token (only when all required vars are present) +resource "enos_local_exec" "docker_create_namespace" { + count = var.vault_address != null && var.vault_root_token != null && var.container_name != null ? 1 : 0 + + inline = [ + <<-EOT + docker exec -e VAULT_ADDR=${var.vault_address} -e VAULT_TOKEN=${var.vault_root_token} \ + ${var.container_name} vault namespace create ${var.namespace_name} + EOT + ] +} + +# Create policy in the namespace +resource "enos_local_exec" "docker_create_policy" { + count = var.vault_address != null && var.vault_root_token != null && var.container_name != null ? 1 : 0 + + inline = [ + <<-EOT + # Write policy to a temp file in the container + docker exec ${var.container_name} sh -c 'cat > /tmp/${var.namespace_name}-policy.hcl << EOF +path "*" { + capabilities = ["create", "read", "update", "delete", "list", "sudo"] +} +EOF' + + # Apply the policy in the namespace + docker exec -e VAULT_ADDR=${var.vault_address} -e VAULT_TOKEN=${var.vault_root_token} -e VAULT_NAMESPACE=${var.namespace_name} \ + ${var.container_name} vault policy write ${var.namespace_name}-policy /tmp/${var.namespace_name}-policy.hcl + EOT + ] + + depends_on = [enos_local_exec.docker_create_namespace] +} + +# Create token in the namespace +resource "enos_local_exec" "docker_create_token" { + count = var.vault_address != null && var.vault_root_token != null && var.container_name != null ? 1 : 0 + + inline = [ + <<-EOT + docker exec -e VAULT_ADDR=${var.vault_address} -e VAULT_TOKEN=${var.vault_root_token} -e VAULT_NAMESPACE=${var.namespace_name} \ + ${var.container_name} vault token create \ + -policy=${var.namespace_name}-policy \ + -ttl=24h \ + -renewable=true \ + -metadata="purpose=${var.namespace_name}-token" \ + -metadata="created_by=docker_namespace_token_module" \ + -format=json | jq -r '.auth.client_token' + EOT + ] + + depends_on = [enos_local_exec.docker_create_policy] +} + +locals { + # For Docker: use the created namespace token, for HCP: use root token (fallback) + namespace_token = length(enos_local_exec.docker_create_token) > 0 ? trimspace(enos_local_exec.docker_create_token[0].stdout) : var.vault_root_token +} + +output "created_at" { + value = timestamp() +} + +output "id" { + value = "docker-${var.namespace_name}-token" +} + +output "token" { + value = local.namespace_token + sensitive = true +} + +output "namespace" { + value = length(enos_local_exec.docker_create_token) > 0 ? var.namespace_name : "root" + description = "The namespace where the token is valid" +} + +output "policy" { + value = length(enos_local_exec.docker_create_token) > 0 ? "${var.namespace_name}-policy" : "root" + description = "The policy assigned to the token" +} diff --git a/enos/modules/docker_network/main.tf b/enos/modules/docker_network/main.tf new file mode 100644 index 0000000000..bea258681d --- /dev/null +++ b/enos/modules/docker_network/main.tf @@ -0,0 +1,31 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +terraform { + required_providers { + docker = { + source = "kreuzwerker/docker" + version = "~> 3.0" + } + } +} + +variable "network_name" { + type = string + description = "The name of the Docker network to create" + default = "vault_cluster" +} + +resource "docker_network" "cluster" { + name = var.network_name +} + +output "network_id" { + value = docker_network.cluster.id + description = "The ID of the created Docker network" +} + +output "network_name" { + value = docker_network.cluster.name + description = "The name of the created Docker network" +} diff --git a/enos/modules/hcp/create_admin_token/main.tf b/enos/modules/hcp/create_admin_token/main.tf index 7396c2cfb2..e7acba349e 100644 --- a/enos/modules/hcp/create_admin_token/main.tf +++ b/enos/modules/hcp/create_admin_token/main.tf @@ -6,6 +6,20 @@ variable "cluster_id" { type = string } +# Docker compatibility variables (accepted but ignored) +variable "vault_root_token" { + description = "Ignored - for Docker compatibility only" + type = string + default = null + sensitive = true +} + +variable "vault_address" { + description = "Ignored - for Docker compatibility only" + type = string + default = null +} + resource "hcp_vault_cluster_admin_token" "token" { cluster_id = var.cluster_id } diff --git a/enos/modules/hcp/create_vault_cluster/main.tf b/enos/modules/hcp/create_vault_cluster/main.tf index 2ec3ee728b..1f628215ec 100644 --- a/enos/modules/hcp/create_vault_cluster/main.tf +++ b/enos/modules/hcp/create_vault_cluster/main.tf @@ -52,6 +52,44 @@ variable "upgrade_type" { default = "MANUAL" } +# Docker-specific variables (ignored but accepted for compatibility) +variable "vault_edition" { + type = string + description = "Ignored - for Docker compatibility only" + default = "" +} + +variable "vault_license" { + type = string + description = "Ignored - for Docker compatibility only" + default = "" + sensitive = true +} + +variable "network_name" { + type = string + description = "Ignored - for Docker compatibility only" + default = "" +} + +variable "cluster_name" { + type = string + description = "Ignored - for Docker compatibility only" + default = "" +} + +variable "use_local_build" { + type = bool + description = "Ignored - for Docker compatibility only" + default = false +} + +variable "local_build_path" { + type = string + description = "Ignored - for Docker compatibility only" + default = "" +} + data "enos_environment" "localhost" {} resource "random_string" "id" { diff --git a/scripts/ci-helper.sh b/scripts/ci-helper.sh index 06fc280e7d..37a9f81fcb 100755 --- a/scripts/ci-helper.sh +++ b/scripts/ci-helper.sh @@ -13,261 +13,290 @@ export GIT_PAGER=cat # Get the build date from the latest commit since it can be used across all # builds function build_date() { - # It's tricky to do an RFC3339 format in a cross platform way, so we hardcode UTC - : "${DATE_FORMAT:="%Y-%m-%dT%H:%M:%SZ"}" - git show --no-show-signature -s --format=%cd --date=format:"$DATE_FORMAT" HEAD + # It's tricky to do an RFC3339 format in a cross platform way, so we hardcode UTC + : "${DATE_FORMAT:="%Y-%m-%dT%H:%M:%SZ"}" + git show --no-show-signature -s --format=%cd --date=format:"$DATE_FORMAT" HEAD } # Get the revision, which is the latest commit SHA function build_revision() { - git rev-parse HEAD + git rev-parse HEAD } # Determine our repository by looking at our origin URL function repo() { - basename -s .git "$(git config --get remote.origin.url)" + basename -s .git "$(git config --get remote.origin.url)" } # Determine the artifact basename based on metadata function artifact_basename() { - : "${PKG_NAME:="vault"}" - : "${GOOS:=$(go env GOOS)}" - : "${GOARCH:=$(go env GOARCH)}" - : "${VERSION_METADATA:="ce"}" + : "${PKG_NAME:="vault"}" + : "${GOOS:=$(go env GOOS)}" + : "${GOARCH:=$(go env GOARCH)}" + : "${VERSION_METADATA:="ce"}" - : "${VERSION:=""}" - if [ -z "$VERSION" ]; then - echo "You must specify the VERSION variable for this command" >&2 - exit 1 - fi + : "${VERSION:=""}" + if [ -z "$VERSION" ]; then + echo "You must specify the VERSION variable for this command" >&2 + exit 1 + fi - local version - version="$VERSION" - if [ "$VERSION_METADATA" != "ce" ]; then - version="${VERSION}+${VERSION_METADATA}" - fi + local version + version="$VERSION" + if [ "$VERSION_METADATA" != "ce" ]; then + version="${VERSION}+${VERSION_METADATA}" + fi - echo "${PKG_NAME}_${version}_${GOOS}_${GOARCH}" + echo "${PKG_NAME}_${version}_${GOOS}_${GOARCH}" +} + +# Copy binaries from BIN_PATH to TARGET_BIN_PATH +function copy_binary() { + : "${BIN_PATH:="dist/"}" + + if [ -z "${TARGET_BIN_PATH:-}" ]; then + echo "TARGET_BIN_PATH not specified, skipping binary copy" >&2 + return 0 + fi + + echo "--> Copying binary from $BIN_PATH to $TARGET_BIN_PATH" + mkdir -p "$TARGET_BIN_PATH" + if [ -d "$BIN_PATH" ]; then + cp -r "$BIN_PATH"/* "$TARGET_BIN_PATH/" + else + echo "Warning: Source directory $BIN_PATH does not exist" >&2 + return 1 + fi } # Bundle the dist directory into a zip +# Note: This always bundles from dist/, regardless of TARGET_BIN_PATH function bundle() { - : "${BUNDLE_PATH:=$(repo_root)/vault.zip}" - echo "--> Bundling dist/* to $BUNDLE_PATH..." - zip -r -j "$BUNDLE_PATH" dist/ + : "${BUNDLE_PATH:=$(repo_root)/vault.zip}" + : "${BIN_PATH:="dist/"}" + + if [ ! -d "$BIN_PATH" ] || [ -z "$(ls -A "$BIN_PATH" 2>/dev/null)" ]; then + echo "Warning: $BIN_PATH is empty or does not exist, bundle will be empty" >&2 + fi + + echo "--> Bundling $BIN_PATH/* to $BUNDLE_PATH..." + zip -r -j "$BUNDLE_PATH" "$BIN_PATH/" } # Determine the root directory of the repository function repo_root() { - git rev-parse --show-toplevel + git rev-parse --show-toplevel } # Build the UI function build_ui() { - local repo_root - repo_root=$(repo_root) + local repo_root + repo_root=$(repo_root) - pushd "$repo_root" - mkdir -p http/web_ui - popd - pushd "$repo_root/ui" - yarn install - npm rebuild node-sass - yarn run build - popd + pushd "$repo_root" + mkdir -p http/web_ui + popd + pushd "$repo_root/ui" + yarn install + npm rebuild node-sass + yarn run build + popd } # Build Vault function build() { - local revision - local base_version - local build_date - local ldflags - local msg + local revision + local base_version + local build_date + local ldflags + local msg - # Get or set our basic build metadata - revision=$(build_revision) - build_date=$(build_date) - base_version=$(version_base) - version=$(version) - : "${BIN_PATH:="dist/"}" #if not run by actions-go-build (enos local) then set this explicitly - : "${GO_TAGS:=""}" - : "${REMOVE_SYMBOLS:=""}" + # Get or set our basic build metadata + revision=$(build_revision) + build_date=$(build_date) + base_version=$(version_base) + version=$(version) + : "${BIN_PATH:="dist/"}" #if not run by actions-go-build (enos local) then set this explicitly + : "${GO_TAGS:=""}" + : "${REMOVE_SYMBOLS:=""}" - # Generate code but make sure we don't slurp in cross compilation env vars - ( - unset GOOS - unset GOARCH - unset CC - unset CC_FOR_TARGET - go generate ./... - ) + # Generate code but make sure we don't slurp in cross compilation env vars + ( + unset GOOS + unset GOARCH + unset CC + unset CC_FOR_TARGET + go generate ./... + ) - # Build our ldflags - msg="--> Building Vault v$version revision $revision, built $build_date..." + # Build our ldflags + msg="--> Building Vault v$version revision $revision, built $build_date..." - # Keep the symbol and dwarf information by default - if [ -n "$REMOVE_SYMBOLS" ]; then - ldflags="-s -w " - else - ldflags="" - fi + # Keep the symbol and dwarf information by default + if [ -n "$REMOVE_SYMBOLS" ]; then + ldflags="-s -w " + else + ldflags="" + fi - # If you read what happens in the "version" package you will see that the - # "version.Version" symbol is automatically set from the embedded VERSION - # file. So why are we setting it again with linker flags? - # - # Well, some third party security scanners like Trivy attempt to determine a - # Go binaries "version" by reading the embedded debug build info. The main - # module "version" reported there has little to do with what we consider - # Vaults version and is instead what the Go module system considers the - # vault modules "pseudo-version"[0]. - # - # What Go determines as the pseudo-version can be pretty complicated. If you - # tag a commit with a semver-ish tag and push it before you build the binary, - # the "pseudo-version" will be the tag value. But what if you build the binary - # before a commit has an associated tag like we do? Well, it depends. If you - # build a Go binary with "-buildvcs" enabled, the "pseudo-version" reported - # here looks something like: "--+dirty". - # If Go cannot resolve a prior tag you'll get "v0.0.0" in place of - # "". If you disable "-buildvcs" you'll get "devel". - # - # As we can see, there's quite a lot of variance in this system and a modules - # "version" is an unreliable way to reason about a softwares "version". But - # that doesn't stop tools from using it and reporting CVEs based on it! - # - # That's right. If you publish a binary with the "+dirty" style pseudo-version, - # and the prior tag that is resolves is associated with a CVE, your binary will - # be flagged for the same CVE even if it has nothing to do with the prior tag. - # If you disable "buildvcs" (we do) these tools cannot determine a "version" - # (because it's always "devel"). When this happens these scanners also fail - # because they can't determine a version. Cool. - # - # So that brings us back to our original query: what's going on with the - # ldflags. To work around this problem, Trivy *reads arbitrary ldflags in - # the binary build info* to determine the "version"![1] when the main module - # does not report a version. And it is because of that, dear reader, that we - # inject our version again via linker flags, to please tooling that relies on - # the unreliable. - # - # [1]: https://go.dev/doc/modules/version-numbers#pseudo-version-number - # [0]: https://trivy.dev/v0.62/docs/coverage/language/golang/#main-module - ldflags="${ldflags} -X github.com/hashicorp/vault/version.GitCommit=$revision -X github.com/hashicorp/vault/version.BuildDate=$build_date -X github.com/hashicorp/vault/version.Version=$base_version" + # If you read what happens in the "version" package you will see that the + # "version.Version" symbol is automatically set from the embedded VERSION + # file. So why are we setting it again with linker flags? + # + # Well, some third party security scanners like Trivy attempt to determine a + # Go binaries "version" by reading the embedded debug build info. The main + # module "version" reported there has little to do with what we consider + # Vaults version and is instead what the Go module system considers the + # vault modules "pseudo-version"[0]. + # + # What Go determines as the pseudo-version can be pretty complicated. If you + # tag a commit with a semver-ish tag and push it before you build the binary, + # the "pseudo-version" will be the tag value. But what if you build the binary + # before a commit has an associated tag like we do? Well, it depends. If you + # build a Go binary with "-buildvcs" enabled, the "pseudo-version" reported + # here looks something like: "--+dirty". + # If Go cannot resolve a prior tag you'll get "v0.0.0" in place of + # "". If you disable "-buildvcs" you'll get "devel". + # + # As we can see, there's quite a lot of variance in this system and a modules + # "version" is an unreliable way to reason about a softwares "version". But + # that doesn't stop tools from using it and reporting CVEs based on it! + # + # That's right. If you publish a binary with the "+dirty" style pseudo-version, + # and the prior tag that is resolves is associated with a CVE, your binary will + # be flagged for the same CVE even if it has nothing to do with the prior tag. + # If you disable "buildvcs" (we do) these tools cannot determine a "version" + # (because it's always "devel"). When this happens these scanners also fail + # because they can't determine a version. Cool. + # + # So that brings us back to our original query: what's going on with the + # ldflags. To work around this problem, Trivy *reads arbitrary ldflags in + # the binary build info* to determine the "version"![1] when the main module + # does not report a version. And it is because of that, dear reader, that we + # inject our version again via linker flags, to please tooling that relies on + # the unreliable. + # + # [1]: https://go.dev/doc/modules/version-numbers#pseudo-version-number + # [0]: https://trivy.dev/v0.62/docs/coverage/language/golang/#main-module + ldflags="${ldflags} -X github.com/hashicorp/vault/version.GitCommit=$revision -X github.com/hashicorp/vault/version.BuildDate=$build_date -X github.com/hashicorp/vault/version.Version=$base_version" - if [[ ${VERSION_METADATA+x} ]]; then - msg="${msg}, metadata ${VERSION_METADATA}" - ldflags="${ldflags} -X github.com/hashicorp/vault/version.VersionMetadata=$VERSION_METADATA" - fi + if [[ ${VERSION_METADATA+x} ]]; then + msg="${msg}, metadata ${VERSION_METADATA}" + ldflags="${ldflags} -X github.com/hashicorp/vault/version.VersionMetadata=$VERSION_METADATA" + fi - # Build vault - echo "$msg" - pushd "$(repo_root)" - mkdir -p dist - mkdir -p out - set -x - go env - go build -v -buildvcs=false -tags "$GO_TAGS" -ldflags "$ldflags" -o dist/ - set +x - popd + # Build vault + echo "$msg" + pushd "$(repo_root)" + mkdir -p dist + mkdir -p out + set -x + go env + go build -v -buildvcs=false -tags "$GO_TAGS" -ldflags "$ldflags" -o dist/ + set +x + popd } # ENT: Prepare legal requirements for packaging function prepare_ent_legal() { - : "${PKG_NAME:="vault"}" + : "${PKG_NAME:="vault"}" - if [ -z "${LICENSE_DIR:-}" ]; then - echo "You must set LICENSE_DIR; example: export LICENSE_DIR=.release/ibm-pao/license/default" 1>&2 - return 1 - fi + if [ -z "${LICENSE_DIR:-}" ]; then + echo "You must set LICENSE_DIR; example: export LICENSE_DIR=.release/ibm-pao/license/default" 1>&2 + return 1 + fi - pushd "$(repo_root)" - mkdir -p dist - cp -R "$LICENSE_DIR" dist/ - mkdir -p ".release/linux/package/usr/share/doc/$PKG_NAME" - cp -R "$LICENSE_DIR" ".release/linux/package/usr/share/doc/$PKG_NAME/" - popd + pushd "$(repo_root)" + mkdir -p dist + cp -R "$LICENSE_DIR" dist/ + mkdir -p ".release/linux/package/usr/share/doc/$PKG_NAME" + cp -R "$LICENSE_DIR" ".release/linux/package/usr/share/doc/$PKG_NAME/" + popd } # CE: Prepare legal requirements for packaging function prepare_ce_legal() { - : "${PKG_NAME:="vault"}" + : "${PKG_NAME:="vault"}" - pushd "$(repo_root)" + pushd "$(repo_root)" - mkdir -p dist - cp LICENSE dist/LICENSE.txt + mkdir -p dist + cp LICENSE dist/LICENSE.txt - mkdir -p ".release/linux/package/usr/share/doc/$PKG_NAME" - cp LICENSE ".release/linux/package/usr/share/doc/$PKG_NAME/LICENSE.txt" + mkdir -p ".release/linux/package/usr/share/doc/$PKG_NAME" + cp LICENSE ".release/linux/package/usr/share/doc/$PKG_NAME/LICENSE.txt" - popd + popd } # version returns the $VAULT_VERSION env variable or reads the VERSION file. function version() { - if [[ -n "${VAULT_VERSION+x}" ]]; then - echo "${VAULT_VERSION}" - return 0 - fi + if [[ -n "${VAULT_VERSION+x}" ]]; then + echo "${VAULT_VERSION}" + return 0 + fi - cat "$(readlink -f "$(dirname "$0")/../version/VERSION")" + cat "$(readlink -f "$(dirname "$0")/../version/VERSION")" } # Base version converts a vault version string into the base version, which omits # any prerelease or edition metadata. function version_base() { - local ver - ver=$(version) - echo "${ver%%-*}" + local ver + ver=$(version) + echo "${ver%%-*}" } # Package version converts a vault version string into a compatible representation for system # packages. function version_package() { - awk '{ gsub("-","~",$1); print $1 }' <<< "$(version)" + awk '{ gsub("-","~",$1); print $1 }' <<<"$(version)" } # Run the CI Helper function main() { - case $1 in - artifact-basename) - artifact_basename - ;; - build) - build - ;; - build-ui) - build_ui - ;; - bundle) - bundle - ;; - date) - build_date - ;; - prepare-ent-legal) - prepare_ent_legal - ;; - prepare-ce-legal) - prepare_ce_legal - ;; - revision) - build_revision - ;; + case $1 in + artifact-basename) + artifact_basename + ;; + build) + build + ;; + build-ui) + build_ui + ;; + bundle) + bundle + ;; + copy-binary) + copy_binary + ;; + date) + build_date + ;; + prepare-ent-legal) + prepare_ent_legal + ;; + prepare-ce-legal) + prepare_ce_legal + ;; + revision) + build_revision + ;; version) - version - ;; - version-base) - version_base - ;; - version-package) - version_package - ;; - *) - echo "unknown sub-command" >&2 - exit 1 - ;; - esac + version + ;; + version-base) + version_base + ;; + version-package) + version_package + ;; + *) + echo "unknown sub-command" >&2 + exit 1 + ;; + esac } main "$@"