VAULT-43442: Adding Enos SDK AWS test to add/delete Vault AWS Roles (#14248) (#14358)

* updating matrix workflow format for easier visualization

* adding test to create and delete Vault AWS Roles

* refactoring functions

* testing pipeline

* testing pipeline

* testing pipeline

* testing pipeline

* finishing up role deletion test

* finishing up role deletion test

Co-authored-by: Tin Vo <tintvo08@gmail.com>
This commit is contained in:
Vault Automation 2026-04-28 14:46:11 -06:00 committed by GitHub
parent 3f3c29607f
commit 06b3374bd5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 220 additions and 74 deletions

View File

@ -112,6 +112,7 @@ jobs:
permissions:
id-token: write # vault-auth
contents: read
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
@ -140,6 +141,7 @@ jobs:
kv/data/github/${{ github.repository }}/license license_1 | VAULT_LICENSE;
kv/data/github/${{ github.repository }}/ibm-license license_1 | VAULT_LICENSE_IBM;
kv/data/github/${{ github.repository }}/github-token token | ELEVATED_GITHUB_TOKEN;
- id: secrets
run: |
if [[ "${{ needs.metadata.outputs.is-ent-repo }}" != 'true' ]]; then
@ -175,6 +177,7 @@ jobs:
echo 'vault-license-ibm=${{ steps.vault-secrets.outputs.VAULT_LICENSE_IBM }}'
} | tee -a "$GITHUB_OUTPUT"
fi
- id: env
run: |
# Configure input environment variables.
@ -204,19 +207,24 @@ jobs:
echo 'ENOS_VAR_verify_ldap_secrets_engine=false'
echo 'ENOS_VAR_verify_log_secrets=true'
} | tee -a "$GITHUB_ENV"
- uses: ./.github/actions/set-up-go
with:
github-token: ${{ steps.secrets.outputs.github-token }}
- name: Install LDAP client tools
run: |
sudo apt-get update
sudo apt-get install -y ldap-utils
- uses: ./.github/actions/install-tools
- uses: hashicorp/setup-terraform@5e8dbf3c6d9deaf4193ca7a8fb23f2ac83bb6c85 # v4.0.0
with:
# the Terraform wrapper will break Terraform execution in Enos because
# it changes the output to text when we expect it to be JSON.
terraform_wrapper: false
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0
with:
@ -226,14 +234,17 @@ jobs:
role-to-assume: ${{ steps.secrets.outputs.aws-role-arn }}
role-skip-session-tagging: true
role-duration-seconds: 3600
- uses: hashicorp/action-setup-enos@6ec106c8f809fe645162d73bea565c65f3269907 # v1.52
with:
github-token: ${{ steps.secrets.outputs.github-token }}
- uses: ./.github/actions/create-dynamic-config
with:
github-token: ${{ steps.secrets.outputs.github-token }}
vault-version: ${{ inputs.vault-version }}
vault-edition: ${{ inputs.vault-edition }}
- name: Prepare scenario dependencies
id: prepare_scenario
run: |
@ -248,32 +259,51 @@ jobs:
echo "junit_results_artifact_name=junit-results_$(echo "${{ matrix.scenario.id.filter }}" | sed -e 's/ /_/g' | sed -e 's/:/=/g')"
echo "failure_summary_artifact_name=failure-summary-enos_$(echo "${{ matrix.scenario.id.filter }}" | sed -e 's/ /_/g' | sed -e 's/:/=/g').md"
} >> "$GITHUB_OUTPUT"
- if: contains(inputs.sample-name, 'build')
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: ${{ inputs.build-artifact-name }}
path: ./enos/support/downloads
- if: contains(inputs.sample-name, 'ent')
name: Configure Vault licenses
run: |
echo "${{ steps.secrets.outputs.vault-license }}" > ./enos/support/vault.hclic || true
echo "${{ steps.secrets.outputs.vault-license-ibm }}" > ./enos/support/ibm-pao.lic || true
- if: contains(matrix.scenario.id.filter, 'consul_edition:ent')
name: Configure Consul license
run: |
echo "${{ steps.secrets.outputs.consul-license }}" > ./enos/support/consul.hclic || true
- name: Configure Vault Radar license
run: |
echo "${{ steps.secrets.outputs.radar-license }}" > ./enos/support/vault-radar.hclic || true
- id: launch
name: enos scenario launch ${{ matrix.scenario.id.filter }}
# Continue once and retry to handle occasional blips when creating infrastructure.
continue-on-error: true
run: enos scenario launch --timeout 45m0s --chdir ./enos ${{ matrix.scenario.id.filter }}
- if: steps.launch.outcome == 'failure'
id: launch_retry
name: Retry enos scenario launch ${{ matrix.scenario.id.filter }}
run: enos scenario launch --timeout 45m0s --chdir ./enos ${{ matrix.scenario.id.filter }}
run: |
# Output our state and plan so we can see where we're at before a retry
echo "Current state:"
enos scenario exec --cmd show --chdir ./enos ${{ matrix.scenario.id.filter }}
echo "Current plan:"
enos scenario exec --cmd plan --chdir ./enos ${{ matrix.scenario.id.filter }}
if ! enos scenario launch --timeout 45m0s --chdir ./enos ${{ matrix.scenario.id.filter }}; then
echo "Retry failed!"
echo "Current state:"
enos scenario exec --cmd show --chdir ./enos ${{ matrix.scenario.id.filter }}
echo "Current plan:"
enos scenario exec --cmd plan --chdir ./enos ${{ matrix.scenario.id.filter }}
fi
- name: Upload Debug Data
if: failure()
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
@ -283,16 +313,19 @@ jobs:
path: ${{ env.ENOS_DEBUG_DATA_ROOT_DIR }}
retention-days: 30
continue-on-error: true
- if: ${{ always() }}
id: destroy
name: enos scenario destroy ${{ matrix.scenario.id.filter }}
continue-on-error: true
run: enos scenario destroy --timeout 10m0s --chdir ./enos ${{ matrix.scenario.id.filter }}
- if: steps.destroy.outcome == 'failure'
id: destroy_retry
name: Retry enos scenario destroy ${{ matrix.scenario.id.filter }}
continue-on-error: true
run: enos scenario destroy --timeout 10m0s --chdir ./enos ${{ matrix.scenario.id.filter }}
- name: Upload Test Results
if: always()
id: upload_test_results
@ -303,6 +336,7 @@ jobs:
retention-days: 7
if-no-files-found: ignore
continue-on-error: true
- name: Upload JUnit Test Results
if: always()
id: upload_junit_results
@ -313,6 +347,7 @@ jobs:
retention-days: 7
if-no-files-found: ignore
continue-on-error: true
- name: Check for test results
if: always()
id: check_test_results
@ -323,6 +358,7 @@ jobs:
else
echo "has_results=false" >> "$GITHUB_OUTPUT"
fi
- name: Prepare Test Results Summary
if: always() && steps.check_test_results.outputs.has_results == 'true'
continue-on-error: true
@ -368,6 +404,7 @@ jobs:
else
echo "⚠️ No test results found in /tmp/vault_test_results_*.json" >> "$GITHUB_STEP_SUMMARY"
fi
- name: Upload Failure Summary
if: always() && steps.check_test_results.outputs.has_results == 'true'
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
@ -376,6 +413,7 @@ jobs:
path: ${{ steps.prepare_scenario.outputs.failure_summary_artifact_name }}
if-no-files-found: ignore
continue-on-error: true
- name: Clean up Enos runtime directories
id: cleanup
if: ${{ always() }}
@ -384,9 +422,11 @@ jobs:
rm -rf /tmp/enos*
rm -rf ./enos/support
rm -rf ./enos/.enos
# Send slack notifications to #feed-vault-enos-failures any of our enos scenario commands fail.
# There is an incoming webhook set up on the "Enos Vault Failure Bot" Slackbot:
# https://api.slack.com/apps/A05E31CH1LG/incoming-webhooks
- if: ${{ always() && ! cancelled() }}
name: Notify launch failed
uses: hashicorp/actions-slack-status@1a3f63b30bd476aee1f3bd6f9d8f2aacc4f14d81 # v2.0.1
@ -394,6 +434,7 @@ jobs:
failure-message: "enos scenario launch ${{ matrix.scenario.id.filter}} failed. \nTriggering event: `${{ github.event_name }}` \nActor: `${{ github.actor }}`"
status: ${{ steps.launch.outcome }}
slack-webhook-url: ${{ steps.secrets.outputs.slack-webhook-url }}
- if: ${{ always() && ! cancelled() }}
name: Notify retry launch failed
uses: hashicorp/actions-slack-status@1a3f63b30bd476aee1f3bd6f9d8f2aacc4f14d81 # v2.0.1
@ -401,6 +442,7 @@ jobs:
failure-message: "retry enos scenario launch ${{ matrix.scenario.id.filter}} failed. \nTriggering event: `${{ github.event_name }}` \nActor: `${{ github.actor }}`"
status: ${{ steps.launch_retry.outcome }}
slack-webhook-url: ${{ steps.secrets.outputs.slack-webhook-url }}
- if: ${{ always() && ! cancelled() }}
name: Notify destroy failed
uses: hashicorp/actions-slack-status@1a3f63b30bd476aee1f3bd6f9d8f2aacc4f14d81 # v2.0.1
@ -408,6 +450,7 @@ jobs:
failure-message: "enos scenario destroy ${{ matrix.scenario.id.filter}} failed. \nTriggering event: `${{ github.event_name }}` \nActor: `${{ github.actor }}`"
status: ${{ steps.destroy.outcome }}
slack-webhook-url: ${{ steps.secrets.outputs.slack-webhook-url }}
- if: ${{ always() && ! cancelled() }}
name: Notify retry destroy failed
uses: hashicorp/actions-slack-status@1a3f63b30bd476aee1f3bd6f9d8f2aacc4f14d81 # v2.0.1

View File

@ -59,3 +59,10 @@ func (s *Session) MustReadKV2(mountPath, secretPath string) *api.Secret {
fullPath := path.Join(mountPath, "data", secretPath)
return s.MustRead(fullPath)
}
func (s *Session) MustDelete(path string) {
s.t.Helper()
_, err := s.Client.Logical().Delete(path)
require.NoError(s.t, err)
}

View File

@ -8,6 +8,7 @@ import (
"crypto/rand"
"encoding/hex"
"fmt"
"os"
"strings"
"testing"
"time"
@ -16,9 +17,15 @@ import (
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/service/iam"
"github.com/aws/aws-sdk-go-v2/service/sts"
"github.com/hashicorp/vault/api"
"github.com/hashicorp/vault/sdk/helper/testcluster/blackbox"
)
// getPolicyArnByName returns the ARN for a policy with the given name.
// =============================================================================
// AWS Helper Functions
// =============================================================================
// getPolicyArnByName finds and returns the ARN for an IAM policy by name.
func getPolicyArnByName(ctx context.Context, iamClient *iam.Client, policyName string) (string, error) {
paginator := iam.NewListPoliciesPaginator(iamClient, &iam.ListPoliciesInput{Scope: "All"})
for paginator.HasMorePages() {
@ -35,7 +42,7 @@ func getPolicyArnByName(ctx context.Context, iamClient *iam.Client, policyName s
return "", fmt.Errorf("policy %s not found", policyName)
}
// getRoleArnByName returns the ARN for a role with the given name.
// getRoleArnByName finds and returns the ARN for an IAM role by name.
func getRoleArnByName(ctx context.Context, iamClient *iam.Client, roleName string) (string, error) {
paginator := iam.NewListRolesPaginator(iamClient, &iam.ListRolesInput{})
for paginator.HasMorePages() {
@ -52,7 +59,7 @@ func getRoleArnByName(ctx context.Context, iamClient *iam.Client, roleName strin
return "", fmt.Errorf("role %s not found", roleName)
}
// createTestIAMUser creates a new IAM user with a unique name, attaches the DemoUser policy, and returns the user/access key info and AWS region.
// createTestIAMUser creates a test IAM user with DemoUser policy and returns credentials.
func createTestIAMUser(t *testing.T) (
userName string,
accessKeyID string,
@ -152,29 +159,7 @@ func createTestIAMUser(t *testing.T) (
return userName, accessKeyID, secretAccessKey, demoUserPolicyArn, assumedRoleArn, awsRegion
}
// getAwsUsernameTemplate returns the username template string for Vault AWS config.
func getAwsUsernameTemplate(awsUserName string) string {
const prefix = `{{ if (eq .Type "STS") }}{{ printf "`
const stsSuffix = `-%s-%s" (random 20) (unix_time) | truncate 32 }}{{ else }}{{ printf "`
const iamUserSuffix = `-%s-%s" (unix_time) (random 20) | truncate 60 }}{{ end }}`
return prefix + awsUserName + stsSuffix + awsUserName + iamUserSuffix
}
// getAllowDescribeRegionsPolicy returns a policy document allowing ec2:DescribeRegions.
func getAllowDescribeRegionsPolicy() string {
return `{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": ["ec2:DescribeRegions"],
"Resource": ["*"]
}
]
}`
}
// deleteIAMUserByAccessKey deletes the IAM user that owns the given access key.
// deleteIAMUserByAccessKey finds and deletes the IAM user owning the specified access key.
func deleteIAMUserByAccessKey(t *testing.T, targetAccessKeyID string) {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
@ -263,3 +248,117 @@ func deleteIAMUserByAccessKey(t *testing.T, targetAccessKeyID string) {
}
}
}
// getAwsUsernameTemplate builds a Vault username template for AWS credential generation.
func getAwsUsernameTemplate(awsUserName string) string {
const prefix = `{{ if (eq .Type "STS") }}{{ printf "`
const stsSuffix = `-%s-%s" (random 20) (unix_time) | truncate 32 }}{{ else }}{{ printf "`
const iamUserSuffix = `-%s-%s" (unix_time) (random 20) | truncate 60 }}{{ end }}`
return prefix + awsUserName + stsSuffix + awsUserName + iamUserSuffix
}
// getAllowDescribeRegionsPolicy returns an IAM policy allowing ec2:DescribeRegions.
func getAllowDescribeRegionsPolicy() string {
return `{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": ["ec2:DescribeRegions"],
"Resource": ["*"]
}
]
}`
}
// =============================================================================
// Vault AWS Secrets Engine Helpers
// =============================================================================
// skipIfNoAWSCredentials skips the test if AWS credentials are missing.
func skipIfNoAWSCredentials(t *testing.T) {
t.Helper()
accessKey := os.Getenv("AWS_ACCESS_KEY_ID")
secretKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
if accessKey == "" || secretKey == "" {
t.Skip("AWS credentials not available - skipping AWS secrets engine test")
}
}
// setupAWSSecretsEngine enables and configures AWS secrets engine, returns mount path.
func setupAWSSecretsEngine(t *testing.T, v *blackbox.Session, accessKeyID, secretAccessKey string, usernameTemplate string) string {
t.Helper()
path := fmt.Sprintf("aws-test-%d", time.Now().UnixNano())
t.Logf("Enabling AWS secrets engine at path: %s", path)
v.MustEnableSecretsEngine(path, &api.MountInput{Type: "aws"})
config := map[string]any{
"access_key": accessKeyID,
"secret_key": secretAccessKey,
"region": "us-east-1",
}
if usernameTemplate != "" {
config["username_template"] = usernameTemplate
t.Logf("Configuring AWS secrets engine with username template")
} else {
t.Logf("Configuring AWS secrets engine with root credentials")
}
v.MustWrite(fmt.Sprintf("%s/config/root", path), config)
return path
}
// createVaultAWSRole creates a Vault AWS role with IAM user credential type.
func createVaultAWSRole(t *testing.T, v *blackbox.Session, path, roleName, policyArn string) {
t.Helper()
t.Logf("Creating Vault AWS role: %s", roleName)
v.MustWrite(fmt.Sprintf("%s/roles/%s", path, roleName), map[string]any{
"credential_type": "iam_user",
"permissions_boundary_arn": policyArn,
"policy_document": getAllowDescribeRegionsPolicy(),
})
}
// verifyRoleExists checks that a Vault AWS role exists in the role list.
func verifyRoleExists(t *testing.T, v *blackbox.Session, path, roleName string) {
t.Helper()
roleList := v.MustList(fmt.Sprintf("%s/roles", path))
if roleList == nil || roleList.Data == nil {
t.Fatalf("failed to list roles at path %s", path)
}
roleKeys, ok := roleList.Data["keys"].([]interface{})
if !ok || len(roleKeys) == 0 {
t.Fatalf("no roles found at path %s", path)
}
for _, key := range roleKeys {
if keyStr, ok := key.(string); ok && keyStr == roleName {
return // Role found
}
}
t.Fatalf("role %q not found in list: %v", roleName, roleKeys)
}
// verifyRoleDeleted checks that a Vault AWS role no longer exists in the role list.
func verifyRoleDeleted(t *testing.T, v *blackbox.Session, path, roleName string) {
t.Helper()
t.Logf("Verifying role %q was deleted", roleName)
rolesList := v.MustList(fmt.Sprintf("%s/roles", path))
if rolesList != nil && rolesList.Data != nil {
if keys, ok := rolesList.Data["keys"].([]interface{}); ok {
for _, key := range keys {
if keyStr, ok := key.(string); ok && keyStr == roleName {
t.Fatalf("role %q still exists after deletion", roleName)
}
}
}
}
t.Logf("Successfully verified role %q was deleted", roleName)
}

View File

@ -5,77 +5,41 @@ package aws
import (
"fmt"
"os"
"testing"
"time"
"github.com/hashicorp/vault/api"
"github.com/hashicorp/vault/sdk/helper/testcluster/blackbox"
)
// TestAWS_GenerateNewUser verifies AWS secrets engine can generate IAM user credentials.
// TestAWS_GenerateNewUser tests AWS secrets engine credential generation.
func TestAWS_GenerateNewUser(t *testing.T) {
t.Parallel()
skipIfNoAWSCredentials(t)
v := blackbox.New(t)
accessKey := os.Getenv("AWS_ACCESS_KEY_ID")
secretKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
if accessKey == "" || secretKey == "" {
t.Log("AWS credentials not available - skipping AWS secrets engine test")
t.Skip("AWS credentials not available - skipping AWS secrets engine test")
}
t.Logf("Creating test IAM user via helpers.go...")
// Create test IAM user for Vault configuration
userName, tempAccessKeyId, tempSecretAccessKey, demoUserPolicyArn, _, _ := createTestIAMUser(t)
t.Logf("Created test IAM user: %s", userName)
// Track generated credentials for cleanup
var newAccessKey string
t.Cleanup(func() {
if newAccessKey != "" {
t.Logf("Cleanup: deleting IAM user created by Vault with access key: %s", newAccessKey)
deleteIAMUserByAccessKey(t, newAccessKey)
}
t.Logf("Cleanup: deleting IAM user by initial access key: %s", tempAccessKeyId)
deleteIAMUserByAccessKey(t, tempAccessKeyId)
})
path := fmt.Sprintf("aws-test-%d", time.Now().UnixNano())
t.Logf("Enabling AWS secrets engine at path: %s", path)
v.MustEnableSecretsEngine(path, &api.MountInput{Type: "aws"})
t.Logf("Configuring AWS secrets engine with root credentials and username template for user: %s", userName)
v.MustWrite(fmt.Sprintf("%s/config/root", path), map[string]any{
"access_key": tempAccessKeyId,
"secret_key": tempSecretAccessKey,
"region": "us-east-1",
"username_template": getAwsUsernameTemplate(userName),
})
// Enable and configure AWS secrets engine
path := setupAWSSecretsEngine(t, v, tempAccessKeyId, tempSecretAccessKey, getAwsUsernameTemplate(userName))
// Create Vault role for credential generation
roleName := "aws-enos-role"
t.Logf("Creating Vault AWS role: %s", roleName)
v.MustWrite(fmt.Sprintf("%s/roles/%s", path, roleName), map[string]any{
"credential_type": "iam_user",
"permissions_boundary_arn": demoUserPolicyArn,
"policy_document": getAllowDescribeRegionsPolicy(),
})
t.Logf("Reading and verifying AWS role configuration for role: %s", roleName)
roleResp := v.MustRead(fmt.Sprintf("%s/roles/%s", path, roleName))
if roleResp.Data == nil {
t.Fatal("Expected to read AWS role configuration")
}
t.Logf("Listing AWS roles at path: %s/roles", path)
rolesList := v.MustList(fmt.Sprintf("%s/roles", path))
if rolesList == nil || rolesList.Data == nil {
t.Fatal("No AWS roles created! (rolesList is nil or Data is nil)")
}
roleKeys, ok := rolesList.Data["keys"].([]interface{})
if !ok || len(roleKeys) == 0 {
t.Fatal("No AWS roles created! (rolesList.Data['keys'] is empty or not a slice)")
}
t.Logf("Found AWS roles: %v", roleKeys)
createVaultAWSRole(t, v, path, roleName, demoUserPolicyArn)
verifyRoleExists(t, v, path, roleName)
// Verify username template was configured
t.Logf("Reading root config to verify username template is set correctly")
rootUser := v.MustRead(fmt.Sprintf("%s/config/root", path))
if rootUser == nil || rootUser.Data == nil {
@ -85,6 +49,7 @@ func TestAWS_GenerateNewUser(t *testing.T) {
t.Fatalf("username_template missing in root config: %#v", rootUser)
}
// Generate new IAM user credentials via Vault
t.Logf("Generating new credentials for IAM user using role: %s", roleName)
newUser := v.MustRead(fmt.Sprintf("%s/creds/%s", path, roleName))
if newUser == nil || newUser.Data == nil {
@ -94,9 +59,41 @@ func TestAWS_GenerateNewUser(t *testing.T) {
t.Fatalf("The new access key is empty or is matching the old one: %v", val)
}
// Extract and save access key for cleanup
var ok bool
newAccessKey, ok = newUser.Data["access_key"].(string)
if !ok || newAccessKey == "" {
t.Fatalf("Could not extract access_key from new credentials: %v", newUser.Data["access_key"])
}
t.Logf("Captured Vault-created access key for cleanup: %s", newAccessKey)
}
// TestAWS_CreateDeleteVaultAwsRole tests Vault AWS role lifecycle.
func TestAWS_CreateDeleteVaultAwsRole(t *testing.T) {
t.Parallel()
skipIfNoAWSCredentials(t)
v := blackbox.New(t)
// Create test IAM user for Vault configuration
userName, tempAccessKeyId, tempSecretAccessKey, demoUserPolicyArn, _, _ := createTestIAMUser(t)
t.Logf("Created test IAM user: %s", userName)
t.Cleanup(func() {
t.Logf("Cleanup: deleting IAM user by initial access key: %s", tempAccessKeyId)
deleteIAMUserByAccessKey(t, tempAccessKeyId)
})
// Enable and configure AWS secrets engine
path := setupAWSSecretsEngine(t, v, tempAccessKeyId, tempSecretAccessKey, "")
// Create and verify Vault role exists
roleName := "aws-enos-role"
createVaultAWSRole(t, v, path, roleName, demoUserPolicyArn)
verifyRoleExists(t, v, path, roleName)
// Delete role and verify it's gone
t.Logf("Deleting Vault AWS role: %s", roleName)
v.MustDelete(fmt.Sprintf("%s/roles/%s", path, roleName))
t.Logf("Role deleted at path: %s", roleName)
verifyRoleDeleted(t, v, path, roleName)
}