diff --git a/.github/enos-run-matrices/build-github-oss-linux-amd64-zip.json b/.github/enos-run-matrices/build-github-oss-linux-amd64-zip.json deleted file mode 100644 index 80b3d55212..0000000000 --- a/.github/enos-run-matrices/build-github-oss-linux-amd64-zip.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "include": [ - { - "scenario": "smoke backend:raft consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:crt edition:oss artifact_type:bundle", - "aws_region": "us-east-1", - "test_group": 3 - }, - { - "scenario": "smoke backend:raft consul_version:1.13.4 distro:rhel seal:awskms arch:amd64 artifact_source:crt edition:oss artifact_type:bundle", - "aws_region": "us-west-2", - "test_group": 4 - }, - { - "scenario": "smoke backend:consul consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:crt edition:oss artifact_type:bundle", - "aws_region": "us-east-1", - "test_group": 1 - }, - { - "scenario": "smoke backend:consul consul_version:1.13.4 distro:rhel seal:awskms arch:amd64 artifact_source:crt edition:oss artifact_type:bundle", - "aws_region": "us-west-2", - "test_group": 5 - }, - { - "scenario": "smoke backend:consul consul_version:1.12.7 distro:ubuntu seal:shamir arch:amd64 artifact_source:crt edition:oss artifact_type:bundle", - "aws_region": "us-east-1", - "test_group": 2 - }, - { - "scenario": "upgrade backend:raft consul_version:1.14.2 distro:rhel seal:awskms arch:amd64 artifact_source:crt edition:oss artifact_type:bundle", - "aws_region": "us-west-2", - "test_group": 3 - }, - { - "scenario": "upgrade backend:raft consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:crt edition:oss artifact_type:bundle", - "aws_region": "us-east-1", - "test_group": 5 - }, - { - "scenario": "upgrade backend:consul consul_version:1.14.2 distro:rhel seal:awskms arch:amd64 artifact_source:crt edition:oss artifact_type:bundle", - "aws_region": "us-west-2", - "test_group": 4 - }, - { - "scenario": "upgrade backend:consul consul_version:1.13.4 distro:ubuntu seal:shamir arch:amd64 artifact_source:crt edition:oss artifact_type:bundle", - "aws_region": "us-east-1", - "test_group": 2 - }, - { - "scenario": "upgrade backend:consul consul_version:1.12.7 distro:rhel seal:awskms arch:amd64 artifact_source:crt edition:oss artifact_type:bundle", - "aws_region": "us-west-2", - "test_group": 1 - } - ] -} diff --git a/.github/enos-run-matrices/build-github-oss-linux-arm64-zip.json b/.github/enos-run-matrices/build-github-oss-linux-arm64-zip.json deleted file mode 100644 index a497fb0ebe..0000000000 --- a/.github/enos-run-matrices/build-github-oss-linux-arm64-zip.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "include": [ - { - "scenario": "smoke backend:raft consul_version:1.13.4 distro:rhel seal:shamir arch:arm64 artifact_source:crt edition:oss artifact_type:bundle", - "aws_region": "us-west-2", - "test_group": 1 - }, - { - "scenario": "smoke backend:raft consul_version:1.14.2 distro:ubuntu seal:awskms arch:arm64 artifact_source:crt edition:oss artifact_type:bundle", - "aws_region": "us-east-1", - "test_group": 2 - }, - { - "scenario": "smoke backend:consul consul_version:1.12.7 distro:ubuntu seal:shamir arch:arm64 artifact_source:crt edition:oss artifact_type:bundle", - "aws_region": "us-west-2", - "test_group": 3 - }, - { - "scenario": "smoke backend:consul consul_version:1.14.2 distro:ubuntu seal:shamir arch:arm64 artifact_source:crt edition:oss artifact_type:bundle", - "aws_region": "us-east-1", - "test_group": 4 - }, - { - "scenario": "smoke backend:consul consul_version:1.13.4 distro:rhel seal:awskms arch:arm64 artifact_source:crt edition:oss artifact_type:bundle", - "aws_region": "us-west-2", - "test_group": 5 - }, - { - "scenario": "upgrade backend:raft consul_version:1.14.2 distro:ubuntu seal:shamir arch:arm64 artifact_source:crt edition:oss artifact_type:bundle", - "aws_region": "us-east-1", - "test_group": 1 - }, - { - "scenario": "upgrade backend:raft consul_version:1.14.2 distro:rhel seal:awskms arch:arm64 artifact_source:crt edition:oss artifact_type:bundle", - "aws_region": "us-west-2", - "test_group": 2 - }, - { - "scenario": "upgrade backend:consul consul_version:1.12.7 distro:rhel seal:awskms arch:arm64 artifact_source:crt edition:oss artifact_type:bundle", - "aws_region": "us-east-1", - "test_group": 3 - }, - { - "scenario": "upgrade backend:consul consul_version:1.13.4 distro:ubuntu seal:shamir arch:arm64 artifact_source:crt edition:oss artifact_type:bundle", - "aws_region": "us-west-2", - "test_group": 4 - }, - { - "scenario": "upgrade backend:consul consul_version:1.14.2 distro:rhel seal:awskms arch:arm64 artifact_source:crt edition:oss artifact_type:bundle", - "aws_region": "us-east-1", - "test_group": 5 - } - ] -} diff --git a/.github/enos-run-matrices/enos_release_testing_oss-artifactory-oss-linux-amd64-zip.json b/.github/enos-run-matrices/enos_release_testing_oss-artifactory-oss-linux-amd64-zip.json deleted file mode 100644 index 857677b72f..0000000000 --- a/.github/enos-run-matrices/enos_release_testing_oss-artifactory-oss-linux-amd64-zip.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "include": [ - { - "scenario": "smoke backend:raft consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", - "aws_region": "us-east-1", - "test_group": 2 - }, - { - "scenario": "smoke backend:raft consul_version:1.13.4 distro:rhel seal:awskms arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", - "aws_region": "us-west-2", - "test_group": 1 - }, - { - "scenario": "smoke backend:consul consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", - "aws_region": "us-east-1", - "test_group": 2 - }, - { - "scenario": "smoke backend:consul consul_version:1.13.4 distro:rhel seal:awskms arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", - "aws_region": "us-west-2", - "test_group": 1 - }, - { - "scenario": "smoke backend:consul consul_version:1.12.7 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", - "aws_region": "us-east-1", - "test_group": 2 - }, - { - "scenario": "upgrade backend:raft consul_version:1.14.2 distro:rhel seal:awskms arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", - "aws_region": "us-west-2", - "test_group": 1 - }, - { - "scenario": "upgrade backend:raft consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", - "aws_region": "us-east-1", - "test_group": 2 - }, - { - "scenario": "upgrade backend:consul consul_version:1.14.2 distro:rhel seal:awskms arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", - "aws_region": "us-west-2", - "test_group": 1 - }, - { - "scenario": "upgrade backend:consul consul_version:1.13.4 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", - "aws_region": "us-east-1", - "test_group": 2 - }, - { - "scenario": "upgrade backend:consul consul_version:1.12.7 distro:rhel seal:awskms arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", - "aws_region": "us-west-2", - "test_group": 1 - } - ] -} diff --git a/.github/enos-run-matrices/enos_release_testing_oss-artifactory-oss-linux-arm64-zip.json b/.github/enos-run-matrices/enos_release_testing_oss-artifactory-oss-linux-arm64-zip.json deleted file mode 100644 index 1c67cd3bcf..0000000000 --- a/.github/enos-run-matrices/enos_release_testing_oss-artifactory-oss-linux-arm64-zip.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "include": [ - { - "scenario": "smoke backend:raft consul_version:1.13.4 distro:rhel seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", - "aws_region": "us-west-2", - "test_group": 1 - }, - { - "scenario": "smoke backend:raft consul_version:1.14.2 distro:ubuntu seal:awskms arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", - "aws_region": "us-east-1", - "test_group": 2 - }, - { - "scenario": "smoke backend:consul consul_version:1.12.7 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", - "aws_region": "us-east-1", - "test_group": 1 - }, - { - "scenario": "smoke backend:consul consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", - "aws_region": "us-east-1", - "test_group": 2 - }, - { - "scenario": "smoke backend:consul consul_version:1.13.4 distro:rhel seal:awskms arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", - "aws_region": "us-west-2", - "test_group": 1 - }, - { - "scenario": "upgrade backend:raft consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", - "aws_region": "us-east-1", - "test_group": 2 - }, - { - "scenario": "upgrade backend:raft consul_version:1.14.2 distro:rhel seal:awskms arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", - "aws_region": "us-west-2", - "test_group": 1 - }, - { - "scenario": "upgrade backend:consul consul_version:1.12.7 distro:rhel seal:awskms arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", - "aws_region": "us-west-2", - "test_group": 2 - }, - { - "scenario": "upgrade backend:consul consul_version:1.13.4 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", - "aws_region": "us-east-1", - "test_group": 1 - }, - { - "scenario": "upgrade backend:consul consul_version:1.14.2 distro:rhel seal:awskms arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", - "aws_region": "us-west-2", - "test_group": 2 - } - ] -} diff --git a/.github/workflows/build-vault-oss.yml b/.github/workflows/build-vault-oss.yml index ea7b2cacb5..056abca369 100644 --- a/.github/workflows/build-vault-oss.yml +++ b/.github/workflows/build-vault-oss.yml @@ -9,9 +9,6 @@ name: build_vault on: workflow_call: inputs: - bundle-path: - required: false - type: string cgo-enabled: type: string default: 0 @@ -35,12 +32,7 @@ on: web-ui-cache-key: type: string required: true - vault-base-version: - type: string - required: true - vault-prerelease-version: - type: string - required: true + jobs: build: runs-on: custom-linux-xl-vault-latest diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1ad1a7e06d..076acdccf9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,13 +34,10 @@ jobs: outputs: build-date: ${{ steps.get-metadata.outputs.build-date }} filepath: ${{ steps.generate-metadata-file.outputs.filepath }} - matrix-test-group: ${{ steps.get-metadata.outputs.matrix-test-group }} package-name: ${{ steps.get-metadata.outputs.package-name }} vault-revision: ${{ steps.get-metadata.outputs.vault-revision }} vault-version: ${{ steps.set-product-version.outputs.product-version }} - vault-base-version: ${{ steps.set-product-version.outputs.base-product-version }} - vault-prerelease-version: ${{ steps.set-product-version.outputs.prerelease-product-version }} - vault-minor-version: ${{ steps.set-product-version.outputs.minor-product-version }} + vault-version-package: ${{ steps.get-metadata.outputs.vault-version-package }} steps: - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - name: Ensure Go modules are cached @@ -55,17 +52,13 @@ jobs: - name: Get metadata id: get-metadata env: - # MATRIX_MAX_TEST_GROUPS is required to determine the randomly selected - # test group. It should be set to the highest test_group used in the - # enos-run-matrices. - MATRIX_MAX_TEST_GROUPS: 5 VAULT_VERSION: ${{ steps.set-product-version.outputs.product-version }} run: | # shellcheck disable=SC2129 echo "build-date=$(make ci-get-date)" >> "$GITHUB_OUTPUT" - echo "matrix-test-group=$(make ci-get-matrix-group-id)" >> "$GITHUB_OUTPUT" echo "package-name=vault" >> "$GITHUB_OUTPUT" echo "vault-revision=$(make ci-get-revision)" >> "$GITHUB_OUTPUT" + echo "vault-version-package=$(make ci-get-version-package)" >> "$GITHUB_OUTPUT" - uses: hashicorp/actions-generate-metadata@v1 id: generate-metadata-file with: @@ -134,8 +127,6 @@ jobs: package-name: ${{ needs.product-metadata.outputs.package-name }} web-ui-cache-key: ${{ needs.build-ui.outputs.cache-key }} vault-version: ${{ needs.product-metadata.outputs.vault-version }} - vault-base-version: ${{ needs.product-metadata.outputs.vault-base-version }} - vault-prerelease-version: ${{ needs.product-metadata.outputs.vault-prerelease-version }} secrets: inherit build-linux: @@ -156,8 +147,6 @@ jobs: package-name: ${{ needs.product-metadata.outputs.package-name }} web-ui-cache-key: ${{ needs.build-ui.outputs.cache-key }} vault-version: ${{ needs.product-metadata.outputs.vault-version }} - vault-base-version: ${{ needs.product-metadata.outputs.vault-base-version }} - vault-prerelease-version: ${{ needs.product-metadata.outputs.vault-prerelease-version }} secrets: inherit build-darwin: @@ -179,8 +168,6 @@ jobs: package-name: ${{ needs.product-metadata.outputs.package-name }} web-ui-cache-key: ${{ needs.build-ui.outputs.cache-key }} vault-version: ${{ needs.product-metadata.outputs.vault-version }} - vault-base-version: ${{ needs.product-metadata.outputs.vault-base-version }} - vault-prerelease-version: ${{ needs.product-metadata.outputs.vault-prerelease-version }} secrets: inherit build-docker: @@ -199,7 +186,7 @@ jobs: - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - uses: hashicorp/actions-docker-build@v1 with: - version: "${{ env.version }}" + version: ${{ env.version }} target: default arch: ${{ matrix.arch }} zip_artifact_name: vault_${{ env.version }}_linux_${{ matrix.arch }}.zip @@ -227,6 +214,7 @@ jobs: target: ubi arch: ${{ matrix.arch }} zip_artifact_name: vault_${{ env.version }}_linux_${{ matrix.arch }}.zip + # The redhat_tag differs on CE and ENT editions. Be mindful when resolving merge conflicts. redhat_tag: quay.io/redhat-isv-containers/5f89bb5e0b94cf64cfeb500a:${{ env.version }}-ubi test: @@ -248,19 +236,25 @@ jobs: fail-fast: false matrix: include: - - matrix-file-name: build-github-oss-linux-amd64-zip + - sample-name: build_oss_linux_amd64_deb + build-artifact-name: vault_${{ needs.product-metadata.outputs.vault-version-package }}-1_amd64.deb + - sample-name: build_oss_linux_arm64_deb + build-artifact-name: vault_${{ needs.product-metadata.outputs.vault-version-package }}-1_arm64.deb + - sample-name: build_oss_linux_amd64_rpm + build-artifact-name: vault-${{ needs.product-metadata.outputs.vault-version-package }}-1.x86_64.rpm + - sample-name: build_oss_linux_arm64_rpm + build-artifact-name: vault-${{ needs.product-metadata.outputs.vault-version-package }}-1.aarch64.rpm + - sample-name: build_oss_linux_amd64_zip build-artifact-name: vault_${{ needs.product-metadata.outputs.vault-version }}_linux_amd64.zip - - matrix-file-name: build-github-oss-linux-arm64-zip + - sample-name: build_oss_linux_arm64_zip build-artifact-name: vault_${{ needs.product-metadata.outputs.vault-version }}_linux_arm64.zip with: build-artifact-name: ${{ matrix.build-artifact-name }} - matrix-file-name: ${{ matrix.matrix-file-name }} - matrix-test-group: ${{ needs.product-metadata.outputs.matrix-test-group }} - vault-edition: oss - vault-revision: ${{ needs.product-metadata.outputs.vault-revision }} + sample-max: 1 + sample-name: ${{ matrix.sample-name }} ssh-key-name: ${{ github.event.repository.name }}-ci-ssh-key + vault-revision: ${{ needs.product-metadata.outputs.vault-revision }} vault-version: ${{ needs.product-metadata.outputs.vault-version }} - vault-minor-version: ${{ needs.product-metadata.outputs.vault-minor-version }} secrets: inherit test-docker-k8s: diff --git a/.github/workflows/enos-release-testing-oss.yml b/.github/workflows/enos-release-testing-oss.yml index 0fbf9f2d44..56dddfc4d6 100644 --- a/.github/workflows/enos-release-testing-oss.yml +++ b/.github/workflows/enos-release-testing-oss.yml @@ -17,28 +17,23 @@ jobs: if: ${{ startsWith(github.event.client_payload.payload.branch, 'release/') }} runs-on: ubuntu-latest outputs: - matrix-test-group: ${{ steps.get-metadata.outputs.matrix-test-group }} vault-revision: ${{ steps.get-metadata.outputs.vault-revision }} vault-version: ${{ steps.set-product-version.outputs.product-version }} - vault-base-version: ${{ steps.set-product-version.outputs.base-product-version }} - vault-prerelease-version: ${{ steps.set-product-version.outputs.prerelease-product-version }} - vault-minor-version: ${{ steps.set-product-version.outputs.minor-product-version }} + vault-version-package: ${{ steps.get-metadata.outputs.vault-version-package }} steps: - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 with: # Check out the repository at the same Git SHA that was used to create # the artifacts to get the correct metadata. ref: ${{ github.event.client_payload.payload.sha }} + - name: Set Product version + id: set-product-version + uses: hashicorp/actions-set-product-version@v1 - id: get-metadata - env: - # MATRIX_MAX_TEST_GROUPS is required to determine the randomly selected - # test group. It should be set to the highest test_group used in the - # enos-run-matrices. - MATRIX_MAX_TEST_GROUPS: 2 run: | # shellcheck disable=SC2129 - echo "matrix-test-group=$(make ci-get-matrix-group-id)" >> "$GITHUB_OUTPUT" echo "vault-revision=$(make ci-get-revision)" >> "$GITHUB_OUTPUT" + echo "vault-version-package=$(echo ${{ steps.set-product-version.outputs.product-version }} | awk '{ gsub("-","~",$1); print $1 }')" >> "$GITHUB_OUTPUT" # Get the workflow summary similar to CRT workflows - name: Release Artifact Info run: | @@ -48,10 +43,6 @@ jobs: echo "__Commit:__ ${{ github.event.client_payload.payload.sha }}" >> "$GITHUB_STEP_SUMMARY" echo "" >> "$GITHUB_STEP_SUMMARY" echo "[Build Workflow](https://github.com/${{github.event.client_payload.payload.org}}/${{github.event.client_payload.payload.repo}}/actions/runs/${{github.event.client_payload.payload.buildworkflowid}})" >> "$GITHUB_STEP_SUMMARY" - - name: Set Product version - id: set-product-version - uses: hashicorp/actions-set-product-version@v1 - test: name: Test ${{ matrix.build-artifact-name }} @@ -62,18 +53,24 @@ jobs: fail-fast: false matrix: include: - - matrix-file-name: enos_release_testing_oss-artifactory-oss-linux-amd64-zip + - sample-name: release_oss_linux_amd64_deb + build-artifact-name: vault_${{ needs.product-metadata.outputs.vault-version-package }}-1_amd64.deb + - sample-name: release_oss_linux_arm64_deb + build-artifact-name: vault_${{ needs.product-metadata.outputs.vault-version-package }}-1_arm64.deb + - sample-name: release_oss_linux_amd64_rpm + build-artifact-name: vault_${{ needs.product-metadata.outputs.vault-version-package }}-1.x86_64.rpm + - sample-name: release_oss_linux_arm64_rpm + build-artifact-name: vault_${{ needs.product-metadata.outputs.vault-version-package }}-1.aarch64.rpm + - sample-name: release_oss_linux_amd64_zip build-artifact-name: vault_${{ needs.product-metadata.outputs.vault-version }}_linux_amd64.zip - - matrix-file-name: enos_release_testing_oss-artifactory-oss-linux-arm64-zip + - sample-name: release_oss_linux_arm64_zip build-artifact-name: vault_${{ needs.product-metadata.outputs.vault-version }}_linux_arm64.zip with: build-artifact-name: ${{ matrix.build-artifact-name }} - matrix-file-name: ${{ matrix.matrix-file-name }} - matrix-test-group: ${{ needs.product-metadata.outputs.matrix-test-group }} - vault-edition: oss + sample-max: 2 + sample-name: ${{ matrix.sample-name }} vault-revision: ${{ needs.product-metadata.outputs.vault-revision }} vault-version: ${{ needs.product-metadata.outputs.vault-version }} - vault-minor-version: ${{ needs.product-metadata.outputs.vault-minor-version }} secrets: inherit save-metadata: diff --git a/.github/workflows/test-run-enos-scenario-matrix.yml b/.github/workflows/test-run-enos-scenario-matrix.yml index c216ae7db1..cdd72e72e0 100644 --- a/.github/workflows/test-run-enos-scenario-matrix.yml +++ b/.github/workflows/test-run-enos-scenario-matrix.yml @@ -11,33 +11,15 @@ on: build-artifact-name: required: true type: string - # The base name of the file in ./github/enos-run-matrices that we use to - # determine which scenarios to run for the build artifact. - # - # They are named in the format of: - # $caller_workflow_name-$artifact_source-$vault_edition-$platform-$arch-$packing_type - # - # Where each are: - # caller_workflow_name: the Github Actions workflow that is calling - # this one - # artifact_source: where we're getting the artifact from. Either - # "github" or "artifactory" - # vault_edition: which edition of vault that we're testing. e.g. "oss" - # or "ent" - # platform: the vault binary target platform, e.g. "linux" or "macos" - # arch: the vault binary target architecture, e.g. "arm64" or "amd64" - # packing_type: how vault binary is packaged, e.g. "zip", "deb", "rpm" - # - # Examples: - # build-github-oss-linux-amd64-zip - matrix-file-name: + # The maximum number of scenarios to include in the test sample. + sample-max: + default: 1 + type: number + # The name of the enos scenario sample that defines compatible scenarios we can + # can test with. + sample-name: required: true type: string - # The test group we want to run. This corresponds to the test_group attribute - # defined in the enos-run-matrices files. - matrix-test-group: - default: 0 - type: string runs-on: # NOTE: The value should be JSON encoded as that's the only way we can # pass arrays with workflow_call. @@ -47,16 +29,9 @@ on: ssh-key-name: type: string default: ${{ github.event.repository.name }}-ci-ssh-key - # Which edition of Vault we're using. e.g. "oss", "ent", "ent.hsm.fips1402" - vault-edition: - required: true - type: string vault-version: required: true type: string - vault-minor-version: - required: true - type: string # The Git commit SHA used as the revision when building vault vault-revision: required: true @@ -67,37 +42,34 @@ jobs: runs-on: ${{ fromJSON(inputs.runs-on) }} outputs: build-date: ${{ steps.metadata.outputs.build-date }} - matrix: ${{ steps.metadata.outputs.matrix }} - env: - # Pass the vault edition as VAULT_METADATA so the CI make targets can create - # values that consider the edition. - VAULT_METADATA: ${{ inputs.vault-edition }} - VAULT_VERSION: ${{ inputs.vault-version }} - VAULT_MINOR_VERSION: ${{ inputs.vault-minor-version }} - # Pass in the matrix and matrix group for filtering - MATRIX_FILE: ./.github/enos-run-matrices/${{ inputs.matrix-file-name }}.json - MATRIX_TEST_GROUP: ${{ inputs.matrix-test-group }} + sample: ${{ steps.metadata.outputs.sample }} steps: - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 with: ref: ${{ inputs.vault-revision }} + - uses: hashicorp/action-setup-enos@v1 + with: + github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} - id: metadata run: | echo "build-date=$(make ci-get-date)" >> "$GITHUB_OUTPUT" - filtered="$(make ci-filter-matrix)" - echo "matrix=$filtered" >> "$GITHUB_OUTPUT" + sample="$(enos scenario sample observe ${{ inputs.sample-name }} --chdir ./enos --min 1 --max ${{ inputs.sample-max }} --seed "$(date +%s%N)" --format json | jq -c ".observation.elements")" + echo "sample=$sample" + echo "sample=$sample" >> "$GITHUB_OUTPUT" - # Run the Enos test scenarios + # Run the Enos test scenario(s) run: needs: metadata + name: run ${{ matrix.scenario.id.filter }} strategy: fail-fast: false # don't fail as that can skip required cleanup steps for jobs - matrix: ${{ fromJson(needs.metadata.outputs.matrix) }} - runs-on: ubuntu-latest + matrix: + include: ${{ fromJSON(needs.metadata.outputs.sample) }} + runs-on: ${{ fromJSON(inputs.runs-on) }} env: GITHUB_TOKEN: ${{ secrets.ELEVATED_GITHUB_TOKEN }} # Pass in enos variables - ENOS_VAR_aws_region: ${{ matrix.aws_region }} + ENOS_VAR_aws_region: ${{ matrix.attributes.aws_region }} ENOS_VAR_aws_ssh_keypair_name: ${{ inputs.ssh-key-name }} ENOS_VAR_aws_ssh_private_key_path: ./support/private_key.pem ENOS_VAR_tfc_api_token: ${{ secrets.TF_API_TOKEN }} @@ -121,7 +93,7 @@ jobs: with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_CI }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_CI }} - aws-region: ${{ matrix.aws_region }} + aws-region: ${{ matrix.attributes.aws_region }} role-to-assume: ${{ secrets.AWS_ROLE_ARN_CI }} role-skip-session-tagging: true role-duration-seconds: 3600 @@ -135,12 +107,12 @@ jobs: echo "${{ secrets.SSH_KEY_PRIVATE_CI }}" > "./enos/support/private_key.pem" chmod 600 "./enos/support/private_key.pem" echo "debug_data_artifact_name=enos-debug-data_$(echo "${{ matrix.scenario }}" | sed -e 's/ /_/g' | sed -e 's/:/=/g')" >> "$GITHUB_OUTPUT" - - if: contains(inputs.matrix-file-name, 'github') + - if: contains(inputs.sample-name, 'build') uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 with: name: ${{ inputs.build-artifact-name }} path: ./enos/support/downloads - - if: contains(inputs.matrix-file-name, 'ent') + - if: contains(inputs.sample-name, 'ent') name: Configure Vault license run: echo "${{ secrets.VAULT_LICENSE }}" > ./enos/support/vault.hclic || true - name: Run Enos scenario @@ -148,12 +120,11 @@ jobs: # Continue once and retry to handle occasional blips when creating # infrastructure. continue-on-error: true - run: enos scenario run --timeout 60m0s --chdir ./enos ${{ matrix.scenario }} + run: enos scenario run --timeout 60m0s --chdir ./enos ${{ matrix.scenario.id.filter }} - name: Retry Enos scenario if necessary id: run_retry if: steps.run.outcome == 'failure' - continue-on-error: true - run: enos scenario run --timeout 60m0s --chdir ./enos ${{ matrix.scenario }} + run: enos scenario run --timeout 60m0s --chdir ./enos ${{ matrix.scenario.id.filter }} - name: Upload Debug Data if: failure() uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 @@ -169,7 +140,7 @@ jobs: # With Enos version 0.0.11 the destroy step returns an error if the infrastructure # is already destroyed by enos run. So temporarily setting it to continue on error in GHA continue-on-error: true - run: enos scenario destroy --timeout 60m0s --chdir ./enos ${{ matrix.scenario }} + run: enos scenario destroy --timeout 60m0s --chdir ./enos ${{ matrix.scenario.id.filter }} - name: Clean up Enos runtime directories id: cleanup if: ${{ always() }} @@ -182,7 +153,7 @@ jobs: # There is an incoming webhook set up on the "Enos Vault Failure Bot" Slackbot https://api.slack.com/apps/A05E31CH1LG/incoming-webhooks - name: Send Slack notification on Enos run failure uses: hashicorp/actions-slack-status@v1 - if: ${{ always() }} + if: ${{ always() && ! cancelled() }} with: failure-message: "An Enos scenario `run` failed. \nTriggering event: `${{ github.event_name }}` \nActor: `${{ github.actor }}`" status: ${{ steps.run.outcome }} @@ -190,7 +161,7 @@ jobs: # Send a Slack notification to #feed-vault-enos-failures if the 'run_retry' step fails. - name: Send Slack notification on Enos run_retry failure uses: hashicorp/actions-slack-status@v1 - if: ${{ always() }} + if: ${{ always() && ! cancelled() }} with: failure-message: "An Enos scenario `run_retry` failed. \nTriggering event: `${{ github.event_name }}` \nActor: `${{ github.actor }}`" status: ${{ steps.run_retry.outcome }} @@ -198,7 +169,7 @@ jobs: # Send a Slack notification to #feed-vault-enos-failures if the 'destroy' step fails. - name: Send Slack notification on Enos destroy failure uses: hashicorp/actions-slack-status@v1 - if: ${{ always() }} + if: ${{ always() && ! cancelled() }} with: failure-message: "An Enos scenario `destroy` failed. \nTriggering event: `${{ github.event_name }}` \nActor: `${{ github.actor }}`" status: ${{ steps.destroy.outcome }} diff --git a/Makefile b/Makefile index 670b413f84..0cebdf74b8 100644 --- a/Makefile +++ b/Makefile @@ -113,7 +113,7 @@ vet: echo "and fix them if necessary before submitting the code for reviewal."; \ fi -# deprecations runs staticcheck tool to look for deprecations. Checks entire code to see if it +# deprecations runs staticcheck tool to look for deprecations. Checks entire code to see if it # has deprecated function, variable, constant or field deprecations: bootstrap prep @BUILD_TAGS='$(BUILD_TAGS)' ./scripts/deprecations-checker.sh "" @@ -128,13 +128,13 @@ tools/codechecker/.bin/codechecker: # vet-codechecker runs our custom linters on the test functions. All output gets # piped to revgrep which will only return an error if new piece of code violates -# the check +# the check vet-codechecker: bootstrap tools/codechecker/.bin/codechecker prep @$(GO_CMD) vet -vettool=./tools/codechecker/.bin/codechecker -tags=$(BUILD_TAGS) ./... 2>&1 | revgrep # vet-codechecker runs our custom linters on the test functions. All output gets -# piped to revgrep which will only return an error if new piece of code that is -# not on main violates the check +# piped to revgrep which will only return an error if new piece of code that is +# not on main violates the check ci-vet-codechecker: ci-bootstrap tools/codechecker/.bin/codechecker prep @$(GO_CMD) vet -vettool=./tools/codechecker/.bin/codechecker -tags=$(BUILD_TAGS) ./... 2>&1 | revgrep origin/main @@ -279,7 +279,7 @@ hana-database-plugin: mongodb-database-plugin: @CGO_ENABLED=0 $(GO_CMD) build -o bin/mongodb-database-plugin ./plugins/database/mongodb/mongodb-database-plugin -.PHONY: bin default prep test vet bootstrap ci-bootstrap fmt fmtcheck mysql-database-plugin mysql-legacy-database-plugin cassandra-database-plugin influxdb-database-plugin postgresql-database-plugin mssql-database-plugin hana-database-plugin mongodb-database-plugin ember-dist ember-dist-dev static-dist static-dist-dev assetcheck check-vault-in-path packages build build-ci semgrep semgrep-ci vet-codechecker ci-vet-codechecker +.PHONY: bin default prep test vet bootstrap ci-bootstrap fmt fmtcheck mysql-database-plugin mysql-legacy-database-plugin cassandra-database-plugin influxdb-database-plugin postgresql-database-plugin mssql-database-plugin hana-database-plugin mongodb-database-plugin ember-dist ember-dist-dev static-dist static-dist-dev assetcheck check-vault-in-path packages build build-ci semgrep semgrep-ci vet-codechecker ci-vet-codechecker .NOTPARALLEL: ember-dist ember-dist-dev @@ -293,34 +293,26 @@ ci-build: ci-build-ui: @$(CURDIR)/scripts/ci-helper.sh build-ui -.PHONY: ci-filter-matrix -ci-filter-matrix: - @$(CURDIR)/scripts/ci-helper.sh matrix-filter-file - -.PHONY: ci-get-date -ci-get-date: - @$(CURDIR)/scripts/ci-helper.sh date -.PHONY: ci-get-matrix-group-id -ci-get-matrix-group-id: - @$(CURDIR)/scripts/ci-helper.sh matrix-group-id - -.PHONY: ci-get-revision -ci-get-revision: - @$(CURDIR)/scripts/ci-helper.sh revision - -.PHONY: ci-prepare-legal -ci-prepare-legal: - @$(CURDIR)/scripts/ci-helper.sh prepare-legal - -.PHONY: ci-get-version-package -ci-get-version-package: - @$(CURDIR)/scripts/ci-helper.sh version-package +.PHONY: ci-bundle +ci-bundle: + @$(CURDIR)/scripts/ci-helper.sh bundle .PHONY: ci-get-artifact-basename ci-get-artifact-basename: @$(CURDIR)/scripts/ci-helper.sh artifact-basename -.PHONY: ci-bundle -ci-bundle: - @$(CURDIR)/scripts/ci-helper.sh bundle +.PHONY: ci-get-date +ci-get-date: + @$(CURDIR)/scripts/ci-helper.sh date +.PHONY: ci-get-revision +ci-get-revision: + @$(CURDIR)/scripts/ci-helper.sh revision + +.PHONY: ci-get-version-package +ci-get-version-package: + @$(CURDIR)/scripts/ci-helper.sh version-package + +.PHONY: ci-prepare-legal +ci-prepare-legal: + @$(CURDIR)/scripts/ci-helper.sh prepare-legal diff --git a/enos/enos-globals.hcl b/enos/enos-globals.hcl new file mode 100644 index 0000000000..a9543280bd --- /dev/null +++ b/enos/enos-globals.hcl @@ -0,0 +1,32 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +globals { + backend_tag_key = "VaultStorage" + build_tags = { + "oss" = ["ui"] + "ent" = ["ui", "enterprise", "ent"] + "ent.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.fips1402"] + "ent.hsm" = ["ui", "enterprise", "cgo", "hsm", "venthsm"] + "ent.hsm.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.hsm.fips1402"] + } + distro_version = { + "rhel" = var.rhel_distro_version + "ubuntu" = var.ubuntu_distro_version + } + packages = ["jq"] + sample_attributes = { + aws_region = ["us-east-1", "us-west-2"] + } + tags = merge({ + "Project Name" : var.project_name + "Project" : "Enos", + "Environment" : "ci" + }, var.tags) + vault_install_dir_packages = { + rhel = "/bin" + ubuntu = "/usr/bin" + } + vault_license_path = abspath(var.vault_license_path != null ? var.vault_license_path : joinpath(path.root, "./support/vault.hclic")) + vault_tag_key = "Type" // enos_vault_start expects Type as the tag key +} diff --git a/enos/enos-samples-oss-build.hcl b/enos/enos-samples-oss-build.hcl new file mode 100644 index 0000000000..3c39901a62 --- /dev/null +++ b/enos/enos-samples-oss-build.hcl @@ -0,0 +1,142 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +sample "build_oss_linux_amd64_deb" { + attributes = global.sample_attributes + + subset "smoke" { + matrix { + arch = ["amd64"] + artifact_source = ["crt"] + artifact_type = ["package"] + distro = ["ubuntu"] + edition = ["oss"] + } + } + + subset "upgrade" { + matrix { + arch = ["amd64"] + artifact_source = ["crt"] + artifact_type = ["package"] + distro = ["ubuntu"] + edition = ["oss"] + } + } +} + +sample "build_oss_linux_arm64_deb" { + attributes = global.sample_attributes + + subset "smoke" { + matrix { + arch = ["arm64"] + artifact_source = ["crt"] + artifact_type = ["package"] + distro = ["ubuntu"] + edition = ["oss"] + } + } + + subset "upgrade" { + matrix { + arch = ["arm64"] + artifact_source = ["crt"] + artifact_type = ["package"] + distro = ["ubuntu"] + edition = ["oss"] + } + } +} + +sample "build_oss_linux_arm64_rpm" { + attributes = global.sample_attributes + + subset "smoke" { + matrix { + arch = ["arm64"] + artifact_source = ["crt"] + artifact_type = ["package"] + distro = ["rhel"] + edition = ["oss"] + } + } + + subset "upgrade" { + matrix { + arch = ["arm64"] + artifact_source = ["crt"] + artifact_type = ["package"] + distro = ["rhel"] + edition = ["oss"] + } + } +} + +sample "build_oss_linux_amd64_rpm" { + attributes = global.sample_attributes + + subset "smoke" { + matrix { + arch = ["amd64"] + artifact_source = ["crt"] + artifact_type = ["package"] + distro = ["rhel"] + edition = ["oss"] + } + } + + subset "upgrade" { + matrix { + arch = ["amd64"] + artifact_source = ["crt"] + artifact_type = ["package"] + distro = ["rhel"] + edition = ["oss"] + } + } +} + +sample "build_oss_linux_amd64_zip" { + attributes = global.sample_attributes + + subset "smoke" { + matrix { + arch = ["amd64"] + artifact_type = ["bundle"] + artifact_source = ["crt"] + edition = ["oss"] + } + } + + subset "upgrade" { + matrix { + arch = ["amd64"] + artifact_type = ["bundle"] + artifact_source = ["crt"] + edition = ["oss"] + } + } +} + +sample "build_oss_linux_arm64_zip" { + attributes = global.sample_attributes + + subset "smoke" { + matrix { + arch = ["arm64"] + artifact_source = ["crt"] + artifact_type = ["bundle"] + edition = ["oss"] + } + } + + subset "upgrade" { + matrix { + arch = ["arm64"] + artifact_source = ["crt"] + artifact_type = ["bundle"] + edition = ["oss"] + } + } +} diff --git a/enos/enos-samples-oss-release.hcl b/enos/enos-samples-oss-release.hcl new file mode 100644 index 0000000000..80eaaa042a --- /dev/null +++ b/enos/enos-samples-oss-release.hcl @@ -0,0 +1,142 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +sample "release_oss_linux_amd64_deb" { + attributes = global.sample_attributes + + subset "smoke" { + matrix { + arch = ["amd64"] + artifact_source = ["artifactory"] + artifact_type = ["package"] + distro = ["ubuntu"] + edition = ["oss"] + } + } + + subset "upgrade" { + matrix { + arch = ["amd64"] + artifact_source = ["artifactory"] + artifact_type = ["package"] + distro = ["ubuntu"] + edition = ["oss"] + } + } +} + +sample "release_oss_linux_arm64_deb" { + attributes = global.sample_attributes + + subset "smoke" { + matrix { + arch = ["arm64"] + artifact_source = ["artifactory"] + artifact_type = ["package"] + distro = ["ubuntu"] + edition = ["oss"] + } + } + + subset "upgrade" { + matrix { + arch = ["arm64"] + artifact_source = ["artifactory"] + artifact_type = ["package"] + distro = ["ubuntu"] + edition = ["oss"] + } + } +} + +sample "release_oss_linux_arm64_rpm" { + attributes = global.sample_attributes + + subset "smoke" { + matrix { + arch = ["arm64"] + artifact_source = ["artifactory"] + artifact_type = ["package"] + distro = ["rhel"] + edition = ["oss"] + } + } + + subset "upgrade" { + matrix { + arch = ["arm64"] + artifact_source = ["artifactory"] + artifact_type = ["package"] + distro = ["rhel"] + edition = ["oss"] + } + } +} + +sample "release_oss_linux_amd64_rpm" { + attributes = global.sample_attributes + + subset "smoke" { + matrix { + arch = ["amd64"] + artifact_source = ["artifactory"] + artifact_type = ["package"] + distro = ["rhel"] + edition = ["oss"] + } + } + + subset "upgrade" { + matrix { + arch = ["amd64"] + artifact_source = ["artifactory"] + artifact_type = ["package"] + distro = ["rhel"] + edition = ["oss"] + } + } +} + +sample "release_oss_linux_amd64_zip" { + attributes = global.sample_attributes + + subset "smoke" { + matrix { + arch = ["amd64"] + artifact_type = ["bundle"] + artifact_source = ["artifactory"] + edition = ["oss"] + } + } + + subset "upgrade" { + matrix { + arch = ["amd64"] + artifact_type = ["bundle"] + artifact_source = ["artifactory"] + edition = ["oss"] + } + } +} + +sample "release_oss_linux_arm64_zip" { + attributes = global.sample_attributes + + subset "smoke" { + matrix { + arch = ["arm64"] + artifact_source = ["artifactory"] + artifact_type = ["bundle"] + edition = ["oss"] + } + } + + subset "upgrade" { + matrix { + arch = ["arm64"] + artifact_source = ["artifactory"] + artifact_type = ["bundle"] + edition = ["oss"] + } + } +} diff --git a/enos/enos-scenario-agent.hcl b/enos/enos-scenario-agent.hcl index f88dd2c032..0988e37bdf 100644 --- a/enos/enos-scenario-agent.hcl +++ b/enos/enos-scenario-agent.hcl @@ -7,6 +7,18 @@ scenario "agent" { artifact_source = ["local", "crt", "artifactory"] distro = ["ubuntu", "rhel"] edition = ["oss", "ent", "ent.fips1402", "ent.hsm", "ent.hsm.fips1402"] + + # Our local builder always creates bundles + exclude { + artifact_source = ["local"] + artifact_type = ["package"] + } + + # HSM and FIPS 140-2 are only supported on amd64 + exclude { + arch = ["arm64"] + edition = ["ent.fips1402", "ent.hsm", "ent.hsm.fips1402"] + } } terraform_cli = terraform_cli.default @@ -18,38 +30,19 @@ scenario "agent" { ] locals { - build_tags = { - "oss" = ["ui"] - "ent" = ["ui", "enterprise", "ent"] - "ent.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.fips1402"] - "ent.hsm" = ["ui", "enterprise", "cgo", "hsm", "venthsm"] - "ent.hsm.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.hsm.fips1402"] - } bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_artifact_path) : null - distro_version = { - "rhel" = var.rhel_distro_version - "ubuntu" = var.ubuntu_distro_version - } enos_provider = { rhel = provider.enos.rhel ubuntu = provider.enos.ubuntu } install_artifactory_artifact = local.bundle_path == null - packages = ["jq"] - tags = merge({ - "Project Name" : var.project_name - "Project" : "Enos", - "Environment" : "ci" - }, var.tags) - vault_license_path = abspath(var.vault_license_path != null ? var.vault_license_path : joinpath(path.root, "./support/vault.hclic")) - vault_tag_key = "Type" // enos_vault_start expects Type as the tag key } step "build_vault" { module = "build_${matrix.artifact_source}" variables { - build_tags = var.vault_local_build_tags != null ? var.vault_local_build_tags : local.build_tags[matrix.edition] + build_tags = var.vault_local_build_tags != null ? var.vault_local_build_tags : global.build_tags[matrix.edition] bundle_path = local.bundle_path goarch = matrix.arch goos = "linux" @@ -74,7 +67,7 @@ scenario "agent" { module = module.create_vpc variables { - common_tags = local.tags + common_tags = global.tags } } @@ -83,7 +76,7 @@ scenario "agent" { module = module.read_license variables { - file_name = local.vault_license_path + file_name = global.vault_license_path } } @@ -96,10 +89,10 @@ scenario "agent" { } variables { - ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][local.distro_version[matrix.distro]] + ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]] awskms_unseal_key_arn = step.create_vpc.kms_key_arn - cluster_tag_key = local.vault_tag_key - common_tags = local.tags + cluster_tag_key = global.vault_tag_key + common_tags = global.tags vpc_id = step.create_vpc.vpc_id } } @@ -123,7 +116,7 @@ scenario "agent" { install_dir = var.vault_install_dir license = matrix.edition != "oss" ? step.read_license.license : null local_artifact_path = local.bundle_path - packages = local.packages + packages = global.packages storage_backend = "raft" target_hosts = step.create_vault_cluster_targets.hosts unseal_method = "shamir" diff --git a/enos/enos-scenario-autopilot.hcl b/enos/enos-scenario-autopilot.hcl index 0f63ffaccf..d8b82f2d09 100644 --- a/enos/enos-scenario-autopilot.hcl +++ b/enos/enos-scenario-autopilot.hcl @@ -10,17 +10,17 @@ scenario "autopilot" { edition = ["ent", "ent.fips1402", "ent.hsm", "ent.hsm.fips1402"] seal = ["awskms", "shamir"] - # Packages are not offered for the oss, ent.fips1402, and ent.hsm.fips1402 editions - exclude { - edition = ["oss", "ent.fips1402", "ent.hsm.fips1402"] - artifact_type = ["package"] - } - # Our local builder always creates bundles exclude { artifact_source = ["local"] artifact_type = ["package"] } + + # HSM and FIPS 140-2 are only supported on amd64 + exclude { + arch = ["arm64"] + edition = ["ent.fips1402", "ent.hsm", "ent.hsm.fips1402"] + } } terraform_cli = terraform_cli.default @@ -32,42 +32,21 @@ scenario "autopilot" { ] locals { - build_tags = { - "ent" = ["ui", "enterprise", "ent"] - "ent.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.fips1402"] - "ent.hsm" = ["ui", "enterprise", "cgo", "hsm", "venthsm"] - "ent.hsm.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.hsm.fips1402"] - } - bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_artifact_path) : null - distro_version = { - "rhel" = var.rhel_distro_version - "ubuntu" = var.ubuntu_distro_version - } + artifact_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_artifact_path) : null enos_provider = { rhel = provider.enos.rhel ubuntu = provider.enos.ubuntu } - packages = ["jq"] - tags = merge({ - "Project Name" : var.project_name - "Project" : "Enos", - "Environment" : "ci" - }, var.tags) - vault_license_path = abspath(var.vault_license_path != null ? var.vault_license_path : joinpath(path.root, "./support/vault.hclic")) - vault_install_dir_packages = { - rhel = "/bin" - ubuntu = "/usr/bin" - } - vault_install_dir = matrix.artifact_type == "bundle" ? var.vault_install_dir : local.vault_install_dir_packages[matrix.distro] - vault_tag_key = "Type" // enos_vault_start expects Type as the tag key + manage_service = matrix.artifact_type == "bundle" + vault_install_dir = matrix.artifact_type == "bundle" ? var.vault_install_dir : global.vault_install_dir_packages[matrix.distro] } step "build_vault" { module = "build_${matrix.artifact_source}" variables { - build_tags = var.vault_local_build_tags != null ? var.vault_local_build_tags : local.build_tags[matrix.edition] - bundle_path = local.bundle_path + build_tags = var.vault_local_build_tags != null ? var.vault_local_build_tags : global.build_tags[matrix.edition] + artifact_path = local.artifact_path goarch = matrix.arch goos = "linux" artifactory_host = matrix.artifact_source == "artifactory" ? var.artifactory_host : null @@ -91,7 +70,7 @@ scenario "autopilot" { module = module.create_vpc variables { - common_tags = local.tags + common_tags = global.tags } } @@ -99,7 +78,7 @@ scenario "autopilot" { module = module.read_license variables { - file_name = local.vault_license_path + file_name = global.vault_license_path } } @@ -112,10 +91,10 @@ scenario "autopilot" { } variables { - ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][local.distro_version[matrix.distro]] + ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]] awskms_unseal_key_arn = step.create_vpc.kms_key_arn - cluster_tag_key = local.vault_tag_key - common_tags = local.tags + cluster_tag_key = global.vault_tag_key + common_tags = global.tags vpc_id = step.create_vpc.vpc_id } } @@ -136,7 +115,7 @@ scenario "autopilot" { cluster_name = step.create_vault_cluster_targets.cluster_name install_dir = local.vault_install_dir license = matrix.edition != "oss" ? step.read_license.license : null - packages = local.packages + packages = global.packages release = var.vault_autopilot_initial_release storage_backend = "raft" storage_backend_addl_config = { @@ -205,9 +184,9 @@ scenario "autopilot" { } variables { - ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][local.distro_version[matrix.distro]] + ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]] awskms_unseal_key_arn = step.create_vpc.kms_key_arn - common_tags = local.tags + common_tags = global.tags cluster_name = step.create_vault_cluster_targets.cluster_name vpc_id = step.create_vpc.vpc_id } @@ -235,8 +214,9 @@ scenario "autopilot" { initialize_cluster = false install_dir = local.vault_install_dir license = matrix.edition != "oss" ? step.read_license.license : null - local_artifact_path = local.bundle_path - packages = local.packages + local_artifact_path = local.artifact_path + manage_service = local.manage_service + packages = global.packages root_token = step.create_vault_cluster.root_token shamir_unseal_keys = matrix.seal == "shamir" ? step.create_vault_cluster.unseal_keys_hex : null storage_backend = "raft" diff --git a/enos/enos-scenario-proxy.hcl b/enos/enos-scenario-proxy.hcl index 520c368943..6595ed40be 100644 --- a/enos/enos-scenario-proxy.hcl +++ b/enos/enos-scenario-proxy.hcl @@ -18,32 +18,11 @@ scenario "proxy" { ] locals { - backend_tag_key = "VaultStorage" - build_tags = { - "oss" = ["ui"] - "ent" = ["ui", "enterprise", "ent"] - "ent.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.fips1402"] - "ent.hsm" = ["ui", "enterprise", "cgo", "hsm", "venthsm"] - "ent.hsm.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.hsm.fips1402"] - } bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_artifact_path) : null - distro_version = { - "rhel" = var.rhel_distro_version - "ubuntu" = var.ubuntu_distro_version - } enos_provider = { rhel = provider.enos.rhel ubuntu = provider.enos.ubuntu } - install_artifactory_artifact = local.bundle_path == null - packages = ["jq"] - tags = merge({ - "Project Name" : var.project_name - "Project" : "Enos", - "Environment" : "ci" - }, var.tags) - vault_license_path = abspath(var.vault_license_path != null ? var.vault_license_path : joinpath(path.root, "./support/vault.hclic")) - vault_tag_key = "Type" // enos_vault_start expects Type as the tag key } step "get_local_metadata" { @@ -55,7 +34,7 @@ scenario "proxy" { module = "build_${matrix.artifact_source}" variables { - build_tags = var.vault_local_build_tags != null ? var.vault_local_build_tags : local.build_tags[matrix.edition] + build_tags = var.vault_local_build_tags != null ? var.vault_local_build_tags : global.build_tags[matrix.edition] bundle_path = local.bundle_path goarch = matrix.arch goos = "linux" @@ -80,7 +59,7 @@ scenario "proxy" { module = module.create_vpc variables { - common_tags = local.tags + common_tags = global.tags } } @@ -89,7 +68,7 @@ scenario "proxy" { module = module.read_license variables { - file_name = local.vault_license_path + file_name = global.vault_license_path } } @@ -102,10 +81,10 @@ scenario "proxy" { } variables { - ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][local.distro_version[matrix.distro]] + ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]] awskms_unseal_key_arn = step.create_vpc.kms_key_arn - cluster_tag_key = local.vault_tag_key - common_tags = local.tags + cluster_tag_key = global.vault_tag_key + common_tags = global.tags vpc_id = step.create_vpc.vpc_id } } @@ -129,7 +108,7 @@ scenario "proxy" { install_dir = var.vault_install_dir license = matrix.edition != "oss" ? step.read_license.license : null local_artifact_path = local.bundle_path - packages = local.packages + packages = global.packages storage_backend = "raft" target_hosts = step.create_vault_cluster_targets.hosts unseal_method = "shamir" diff --git a/enos/enos-scenario-replication.hcl b/enos/enos-scenario-replication.hcl index bba7c9e9dc..fb645a393b 100644 --- a/enos/enos-scenario-replication.hcl +++ b/enos/enos-scenario-replication.hcl @@ -17,17 +17,17 @@ scenario "replication" { secondary_backend = ["raft", "consul"] secondary_seal = ["awskms", "shamir"] - # Packages are not offered for the oss, ent.fips1402, and ent.hsm.fips1402 editions - exclude { - edition = ["ent.fips1402", "ent.hsm.fips1402"] - artifact_type = ["package"] - } - # Our local builder always creates bundles exclude { artifact_source = ["local"] artifact_type = ["package"] } + + # HSM and FIPS 140-2 are only supported on amd64 + exclude { + arch = ["arm64"] + edition = ["ent.fips1402", "ent.hsm", "ent.hsm.fips1402"] + } } terraform_cli = terraform_cli.default @@ -39,45 +39,21 @@ scenario "replication" { ] locals { - # The path to the backend license file (Consul Enterprise) - backend_license_path = abspath(var.backend_license_path != null ? var.backend_license_path : joinpath(path.root, "./support/consul.hclic")) - backend_tag_key = "VaultStorage" - build_tags = { - "ent" = ["ui", "enterprise", "ent"] - "ent.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.fips1402"] - "ent.hsm" = ["ui", "enterprise", "cgo", "hsm", "venthsm"] - "ent.hsm.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.hsm.fips1402"] - } - distro_version = { - "rhel" = var.rhel_distro_version - "ubuntu" = var.ubuntu_distro_version - } - bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_artifact_path) : null + artifact_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_artifact_path) : null enos_provider = { rhel = provider.enos.rhel ubuntu = provider.enos.ubuntu } - packages = ["jq"] - tags = merge({ - "Project Name" : var.project_name - "Project" : "Enos", - "Environment" : "ci" - }, var.tags) - vault_license_path = abspath(var.vault_license_path != null ? var.vault_license_path : joinpath(path.root, "./support/vault.hclic")) - vault_install_dir_packages = { - rhel = "/bin" - ubuntu = "/usr/bin" - } - vault_install_dir = matrix.artifact_type == "bundle" ? var.vault_install_dir : local.vault_install_dir_packages[matrix.distro] - vault_tag_key = "Type" // enos_vault_start expects Type as the tag key + manage_service = matrix.artifact_type == "bundle" + vault_install_dir = matrix.artifact_type == "bundle" ? var.vault_install_dir : global.vault_install_dir_packages[matrix.distro] } step "build_vault" { module = "build_${matrix.artifact_source}" variables { - build_tags = var.vault_local_build_tags != null ? var.vault_local_build_tags : local.build_tags[matrix.edition] - bundle_path = local.bundle_path + build_tags = var.vault_local_build_tags != null ? var.vault_local_build_tags : global.build_tags[matrix.edition] + artifact_path = local.artifact_path goarch = matrix.arch goos = "linux" artifactory_host = matrix.artifact_source == "artifactory" ? var.artifactory_host : null @@ -101,7 +77,7 @@ scenario "replication" { module = module.create_vpc variables { - common_tags = local.tags + common_tags = global.tags } } @@ -112,7 +88,7 @@ scenario "replication" { module = module.read_license variables { - file_name = local.backend_license_path + file_name = global.backend_license_path } } @@ -136,10 +112,10 @@ scenario "replication" { } variables { - ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][local.distro_version[matrix.distro]] + ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]] awskms_unseal_key_arn = step.create_vpc.kms_key_arn - cluster_tag_key = local.vault_tag_key - common_tags = local.tags + cluster_tag_key = global.vault_tag_key + common_tags = global.tags vpc_id = step.create_vpc.vpc_id } } @@ -157,8 +133,8 @@ scenario "replication" { variables { ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["22.04"] awskms_unseal_key_arn = step.create_vpc.kms_key_arn - cluster_tag_key = local.backend_tag_key - common_tags = local.tags + cluster_tag_key = global.backend_tag_key + common_tags = global.tags vpc_id = step.create_vpc.vpc_id } } @@ -175,11 +151,11 @@ scenario "replication" { } variables { - ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][local.distro_version[matrix.distro]] + ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]] awskms_unseal_key_arn = step.create_vpc.kms_key_arn cluster_name = step.create_primary_cluster_targets.cluster_name - cluster_tag_key = local.vault_tag_key - common_tags = local.tags + cluster_tag_key = global.vault_tag_key + common_tags = global.tags vpc_id = step.create_vpc.vpc_id } } @@ -193,10 +169,10 @@ scenario "replication" { } variables { - ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][local.distro_version[matrix.distro]] + ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]] awskms_unseal_key_arn = step.create_vpc.kms_key_arn - cluster_tag_key = local.vault_tag_key - common_tags = local.tags + cluster_tag_key = global.vault_tag_key + common_tags = global.tags vpc_id = step.create_vpc.vpc_id } } @@ -212,8 +188,8 @@ scenario "replication" { variables { ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["22.04"] awskms_unseal_key_arn = step.create_vpc.kms_key_arn - cluster_tag_key = local.backend_tag_key - common_tags = local.tags + cluster_tag_key = global.backend_tag_key + common_tags = global.tags vpc_id = step.create_vpc.vpc_id } } @@ -230,7 +206,7 @@ scenario "replication" { variables { cluster_name = step.create_primary_cluster_backend_targets.cluster_name - cluster_tag_key = local.backend_tag_key + cluster_tag_key = global.backend_tag_key license = (matrix.primary_backend == "consul" && var.backend_edition == "ent") ? step.read_backend_license.license : null release = { edition = var.backend_edition @@ -256,7 +232,7 @@ scenario "replication" { artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null awskms_unseal_key_arn = step.create_vpc.kms_key_arn backend_cluster_name = step.create_primary_cluster_backend_targets.cluster_name - backend_cluster_tag_key = local.backend_tag_key + backend_cluster_tag_key = global.backend_tag_key consul_license = (matrix.primary_backend == "consul" && var.backend_edition == "ent") ? step.read_backend_license.license : null cluster_name = step.create_primary_cluster_targets.cluster_name consul_release = matrix.primary_backend == "consul" ? { @@ -266,8 +242,9 @@ scenario "replication" { enable_file_audit_device = var.vault_enable_file_audit_device install_dir = local.vault_install_dir license = matrix.edition != "oss" ? step.read_vault_license.license : null - local_artifact_path = local.bundle_path - packages = local.packages + local_artifact_path = local.artifact_path + manage_service = local.manage_service + packages = global.packages storage_backend = matrix.primary_backend target_hosts = step.create_primary_cluster_targets.hosts unseal_method = matrix.primary_seal @@ -286,7 +263,7 @@ scenario "replication" { variables { cluster_name = step.create_secondary_cluster_backend_targets.cluster_name - cluster_tag_key = local.backend_tag_key + cluster_tag_key = global.backend_tag_key license = (matrix.secondary_backend == "consul" && var.backend_edition == "ent") ? step.read_backend_license.license : null release = { edition = var.backend_edition @@ -312,7 +289,7 @@ scenario "replication" { artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null awskms_unseal_key_arn = step.create_vpc.kms_key_arn backend_cluster_name = step.create_secondary_cluster_backend_targets.cluster_name - backend_cluster_tag_key = local.backend_tag_key + backend_cluster_tag_key = global.backend_tag_key consul_license = (matrix.secondary_backend == "consul" && var.backend_edition == "ent") ? step.read_backend_license.license : null cluster_name = step.create_secondary_cluster_targets.cluster_name consul_release = matrix.secondary_backend == "consul" ? { @@ -322,8 +299,9 @@ scenario "replication" { enable_file_audit_device = var.vault_enable_file_audit_device install_dir = local.vault_install_dir license = matrix.edition != "oss" ? step.read_vault_license.license : null - local_artifact_path = local.bundle_path - packages = local.packages + local_artifact_path = local.artifact_path + manage_service = local.manage_service + packages = global.packages storage_backend = matrix.secondary_backend target_hosts = step.create_secondary_cluster_targets.hosts unseal_method = matrix.secondary_seal @@ -553,25 +531,27 @@ scenario "replication" { artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null awskms_unseal_key_arn = step.create_vpc.kms_key_arn backend_cluster_name = step.create_primary_cluster_backend_targets.cluster_name - backend_cluster_tag_key = local.backend_tag_key + backend_cluster_tag_key = global.backend_tag_key cluster_name = step.create_primary_cluster_targets.cluster_name consul_license = (matrix.primary_backend == "consul" && var.backend_edition == "ent") ? step.read_backend_license.license : null consul_release = matrix.primary_backend == "consul" ? { edition = var.backend_edition version = matrix.consul_version } : null - force_unseal = matrix.primary_seal == "shamir" - initialize_cluster = false - install_dir = local.vault_install_dir - license = matrix.edition != "oss" ? step.read_vault_license.license : null - local_artifact_path = local.bundle_path - packages = local.packages - root_token = step.create_primary_cluster.root_token - shamir_unseal_keys = matrix.primary_seal == "shamir" ? step.create_primary_cluster.unseal_keys_hex : null - storage_backend = matrix.primary_backend - storage_node_prefix = "newprimary_node" - target_hosts = step.create_primary_cluster_additional_targets.hosts - unseal_method = matrix.primary_seal + enable_file_audit_device = var.vault_enable_file_audit_device + force_unseal = matrix.primary_seal == "shamir" + initialize_cluster = false + install_dir = local.vault_install_dir + license = matrix.edition != "oss" ? step.read_vault_license.license : null + local_artifact_path = local.artifact_path + manage_service = local.manage_service + packages = global.packages + root_token = step.create_primary_cluster.root_token + shamir_unseal_keys = matrix.primary_seal == "shamir" ? step.create_primary_cluster.unseal_keys_hex : null + storage_backend = matrix.primary_backend + storage_node_prefix = "newprimary_node" + target_hosts = step.create_primary_cluster_additional_targets.hosts + unseal_method = matrix.primary_seal } } diff --git a/enos/enos-scenario-smoke.hcl b/enos/enos-scenario-smoke.hcl index 97a7d2713e..27bc342e4a 100644 --- a/enos/enos-scenario-smoke.hcl +++ b/enos/enos-scenario-smoke.hcl @@ -12,17 +12,17 @@ scenario "smoke" { edition = ["oss", "ent", "ent.fips1402", "ent.hsm", "ent.hsm.fips1402"] seal = ["awskms", "shamir"] - # Packages are not offered for the oss, ent.fips1402, and ent.hsm.fips1402 editions - exclude { - edition = ["oss", "ent.fips1402", "ent.hsm.fips1402"] - artifact_type = ["package"] - } - # Our local builder always creates bundles exclude { artifact_source = ["local"] artifact_type = ["package"] } + + # HSM and FIPS 140-2 are only supported on amd64 + exclude { + arch = ["arm64"] + edition = ["ent.fips1402", "ent.hsm", "ent.hsm.fips1402"] + } } terraform_cli = terraform_cli.default @@ -34,37 +34,13 @@ scenario "smoke" { ] locals { - backend_license_path = abspath(var.backend_license_path != null ? var.backend_license_path : joinpath(path.root, "./support/consul.hclic")) - backend_tag_key = "VaultStorage" - build_tags = { - "oss" = ["ui"] - "ent" = ["ui", "enterprise", "ent"] - "ent.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.fips1402"] - "ent.hsm" = ["ui", "enterprise", "cgo", "hsm", "venthsm"] - "ent.hsm.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.hsm.fips1402"] - } - bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_artifact_path) : null - distro_version = { - "rhel" = var.rhel_distro_version - "ubuntu" = var.ubuntu_distro_version - } + artifact_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_artifact_path) : null enos_provider = { rhel = provider.enos.rhel ubuntu = provider.enos.ubuntu } - packages = ["jq"] - tags = merge({ - "Project Name" : var.project_name - "Project" : "Enos", - "Environment" : "ci" - }, var.tags) - vault_license_path = abspath(var.vault_license_path != null ? var.vault_license_path : joinpath(path.root, "./support/vault.hclic")) - vault_install_dir_packages = { - rhel = "/bin" - ubuntu = "/usr/bin" - } - vault_install_dir = matrix.artifact_type == "bundle" ? var.vault_install_dir : local.vault_install_dir_packages[matrix.distro] - vault_tag_key = "Type" // enos_vault_start expects Type as the tag key + manage_service = matrix.artifact_type == "bundle" + vault_install_dir = matrix.artifact_type == "bundle" ? var.vault_install_dir : global.vault_install_dir_packages[matrix.distro] } step "get_local_metadata" { @@ -76,8 +52,8 @@ scenario "smoke" { module = "build_${matrix.artifact_source}" variables { - build_tags = var.vault_local_build_tags != null ? var.vault_local_build_tags : local.build_tags[matrix.edition] - bundle_path = local.bundle_path + build_tags = var.vault_local_build_tags != null ? var.vault_local_build_tags : global.build_tags[matrix.edition] + artifact_path = local.artifact_path goarch = matrix.arch goos = "linux" artifactory_host = matrix.artifact_source == "artifactory" ? var.artifactory_host : null @@ -101,7 +77,7 @@ scenario "smoke" { module = module.create_vpc variables { - common_tags = local.tags + common_tags = global.tags } } @@ -112,7 +88,7 @@ scenario "smoke" { module = module.read_license variables { - file_name = local.backend_license_path + file_name = global.backend_license_path } } @@ -121,7 +97,7 @@ scenario "smoke" { module = module.read_license variables { - file_name = local.vault_license_path + file_name = global.vault_license_path } } @@ -134,10 +110,10 @@ scenario "smoke" { } variables { - ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][local.distro_version[matrix.distro]] + ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]] awskms_unseal_key_arn = step.create_vpc.kms_key_arn - cluster_tag_key = local.vault_tag_key - common_tags = local.tags + cluster_tag_key = global.vault_tag_key + common_tags = global.tags vpc_id = step.create_vpc.vpc_id } } @@ -153,8 +129,8 @@ scenario "smoke" { variables { ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["22.04"] awskms_unseal_key_arn = step.create_vpc.kms_key_arn - cluster_tag_key = local.backend_tag_key - common_tags = local.tags + cluster_tag_key = global.backend_tag_key + common_tags = global.tags vpc_id = step.create_vpc.vpc_id } } @@ -171,7 +147,7 @@ scenario "smoke" { variables { cluster_name = step.create_vault_cluster_backend_targets.cluster_name - cluster_tag_key = local.backend_tag_key + cluster_tag_key = global.backend_tag_key license = (matrix.backend == "consul" && var.backend_edition == "ent") ? step.read_backend_license.license : null release = { edition = var.backend_edition @@ -197,7 +173,7 @@ scenario "smoke" { artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null awskms_unseal_key_arn = step.create_vpc.kms_key_arn backend_cluster_name = step.create_vault_cluster_backend_targets.cluster_name - backend_cluster_tag_key = local.backend_tag_key + backend_cluster_tag_key = global.backend_tag_key cluster_name = step.create_vault_cluster_targets.cluster_name consul_license = (matrix.backend == "consul" && var.backend_edition == "ent") ? step.read_backend_license.license : null consul_release = matrix.backend == "consul" ? { @@ -207,8 +183,9 @@ scenario "smoke" { enable_file_audit_device = var.vault_enable_file_audit_device install_dir = local.vault_install_dir license = matrix.edition != "oss" ? step.read_vault_license.license : null - local_artifact_path = local.bundle_path - packages = local.packages + local_artifact_path = local.artifact_path + manage_service = local.manage_service + packages = global.packages storage_backend = matrix.backend target_hosts = step.create_vault_cluster_targets.hosts unseal_method = matrix.seal diff --git a/enos/enos-scenario-upgrade.hcl b/enos/enos-scenario-upgrade.hcl index 81b18bdb8f..54b1cc273b 100644 --- a/enos/enos-scenario-upgrade.hcl +++ b/enos/enos-scenario-upgrade.hcl @@ -12,10 +12,16 @@ scenario "upgrade" { edition = ["oss", "ent", "ent.fips1402", "ent.hsm", "ent.hsm.fips1402"] seal = ["awskms", "shamir"] - # Packages are not offered for the oss, ent.fips1402, and ent.hsm.fips1402 editions + # Our local builder always creates bundles exclude { - edition = ["oss", "ent.fips1402", "ent.hsm.fips1402"] - artifact_type = ["package"] + artifact_source = ["local"] + artifact_type = ["package"] + } + + # HSM and FIPS 140-2 are only supported on amd64 + exclude { + arch = ["arm64"] + edition = ["ent.fips1402", "ent.hsm", "ent.hsm.fips1402"] } } @@ -28,37 +34,13 @@ scenario "upgrade" { ] locals { - backend_license_path = abspath(var.backend_license_path != null ? var.backend_license_path : joinpath(path.root, "./support/consul.hclic")) - backend_tag_key = "VaultStorage" - build_tags = { - "oss" = ["ui"] - "ent" = ["ui", "enterprise", "ent"] - "ent.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.fips1402"] - "ent.hsm" = ["ui", "enterprise", "cgo", "hsm", "venthsm"] - "ent.hsm.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.hsm.fips1402"] - } - bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_artifact_path) : null - distro_version = { - "rhel" = var.rhel_distro_version - "ubuntu" = var.ubuntu_distro_version - } + artifact_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_artifact_path) : null enos_provider = { rhel = provider.enos.rhel ubuntu = provider.enos.ubuntu } - packages = ["jq"] - tags = merge({ - "Project Name" : var.project_name - "Project" : "Enos", - "Environment" : "ci" - }, var.tags) - vault_license_path = abspath(var.vault_license_path != null ? var.vault_license_path : joinpath(path.root, "./support/vault.hclic")) - vault_install_dir_packages = { - rhel = "/bin" - ubuntu = "/usr/bin" - } - vault_install_dir = matrix.artifact_type == "bundle" ? var.vault_install_dir : local.vault_install_dir_packages[matrix.distro] - vault_tag_key = "Type" // enos_vault_start expects Type as the tag key + manage_service = matrix.artifact_type == "bundle" + vault_install_dir = matrix.artifact_type == "bundle" ? var.vault_install_dir : global.vault_install_dir_packages[matrix.distro] } step "get_local_metadata" { @@ -71,8 +53,8 @@ scenario "upgrade" { module = "build_${matrix.artifact_source}" variables { - build_tags = var.vault_local_build_tags != null ? var.vault_local_build_tags : local.build_tags[matrix.edition] - bundle_path = local.bundle_path + build_tags = var.vault_local_build_tags != null ? var.vault_local_build_tags : global.build_tags[matrix.edition] + artifact_path = local.artifact_path goarch = matrix.arch goos = "linux" artifactory_host = matrix.artifact_source == "artifactory" ? var.artifactory_host : null @@ -96,7 +78,7 @@ scenario "upgrade" { module = module.create_vpc variables { - common_tags = local.tags + common_tags = global.tags } } @@ -107,7 +89,7 @@ scenario "upgrade" { module = module.read_license variables { - file_name = local.backend_license_path + file_name = global.backend_license_path } } @@ -116,7 +98,7 @@ scenario "upgrade" { module = module.read_license variables { - file_name = local.vault_license_path + file_name = global.vault_license_path } } @@ -129,10 +111,10 @@ scenario "upgrade" { } variables { - ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][local.distro_version[matrix.distro]] + ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]] awskms_unseal_key_arn = step.create_vpc.kms_key_arn - cluster_tag_key = local.vault_tag_key - common_tags = local.tags + cluster_tag_key = global.vault_tag_key + common_tags = global.tags vpc_id = step.create_vpc.vpc_id } } @@ -148,8 +130,8 @@ scenario "upgrade" { variables { ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["22.04"] awskms_unseal_key_arn = step.create_vpc.kms_key_arn - cluster_tag_key = local.backend_tag_key - common_tags = local.tags + cluster_tag_key = global.backend_tag_key + common_tags = global.tags vpc_id = step.create_vpc.vpc_id } } @@ -166,7 +148,7 @@ scenario "upgrade" { variables { cluster_name = step.create_vault_cluster_backend_targets.cluster_name - cluster_tag_key = local.backend_tag_key + cluster_tag_key = global.backend_tag_key license = (matrix.backend == "consul" && var.backend_edition == "ent") ? step.read_backend_license.license : null release = { edition = var.backend_edition @@ -191,7 +173,7 @@ scenario "upgrade" { variables { awskms_unseal_key_arn = step.create_vpc.kms_key_arn backend_cluster_name = step.create_vault_cluster_backend_targets.cluster_name - backend_cluster_tag_key = local.backend_tag_key + backend_cluster_tag_key = global.backend_tag_key consul_license = (matrix.backend == "consul" && var.backend_edition == "ent") ? step.read_backend_license.license : null cluster_name = step.create_vault_cluster_targets.cluster_name consul_release = matrix.backend == "consul" ? { @@ -201,7 +183,7 @@ scenario "upgrade" { enable_file_audit_device = var.vault_enable_file_audit_device install_dir = local.vault_install_dir license = matrix.edition != "oss" ? step.read_vault_license.license : null - packages = local.packages + packages = global.packages release = var.vault_upgrade_initial_release storage_backend = matrix.backend target_hosts = step.create_vault_cluster_targets.hosts @@ -259,7 +241,7 @@ scenario "upgrade" { variables { vault_api_addr = "http://localhost:8200" vault_instances = step.create_vault_cluster_targets.hosts - vault_local_artifact_path = local.bundle_path + vault_local_artifact_path = local.artifact_path vault_artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null vault_install_dir = local.vault_install_dir vault_unseal_keys = matrix.seal == "shamir" ? step.create_vault_cluster.unseal_keys_hex : null diff --git a/enos/modules/vault_cluster/main.tf b/enos/modules/vault_cluster/main.tf index b9203e2c46..8fc34f2b5a 100644 --- a/enos/modules/vault_cluster/main.tf +++ b/enos/modules/vault_cluster/main.tf @@ -69,23 +69,6 @@ locals { vault_service_user = "vault" } -resource "enos_remote_exec" "install_packages" { - for_each = { - for idx, host in var.target_hosts : idx => var.target_hosts[idx] - if length(var.packages) > 0 - } - - content = templatefile("${path.module}/templates/install-packages.sh", { - packages = join(" ", var.packages) - }) - - transport = { - ssh = { - host = each.value.public_ip - } - } -} - resource "enos_bundle_install" "consul" { for_each = { for idx, host in var.target_hosts : idx => var.target_hosts[idx] @@ -117,6 +100,26 @@ resource "enos_bundle_install" "vault" { } } +resource "enos_remote_exec" "install_packages" { + depends_on = [ + enos_bundle_install.vault, // Don't race for the package manager locks with vault install + ] + for_each = { + for idx, host in var.target_hosts : idx => var.target_hosts[idx] + if length(var.packages) > 0 + } + + content = templatefile("${path.module}/templates/install-packages.sh", { + packages = join(" ", var.packages) + }) + + transport = { + ssh = { + host = each.value.public_ip + } + } +} + resource "enos_consul_start" "consul" { for_each = enos_bundle_install.consul @@ -272,6 +275,7 @@ resource "enos_vault_unseal" "leader" { # user on all nodes, since logging will only happen on the leader. resource "enos_remote_exec" "create_audit_log_dir" { depends_on = [ + enos_bundle_install.vault, enos_vault_unseal.leader, ] for_each = toset([ @@ -395,3 +399,11 @@ resource "enos_remote_exec" "vault_write_license" { } } } + +resource "enos_local_exec" "wait_for_install_packages" { + depends_on = [ + enos_remote_exec.install_packages, + ] + + inline = ["true"] +} diff --git a/enos/modules/vault_get_cluster_ips/scripts/get-leader-private-ip.sh b/enos/modules/vault_get_cluster_ips/scripts/get-leader-private-ip.sh index 76b44f5355..7e1655ff84 100644 --- a/enos/modules/vault_get_cluster_ips/scripts/get-leader-private-ip.sh +++ b/enos/modules/vault_get_cluster_ips/scripts/get-leader-private-ip.sh @@ -18,7 +18,7 @@ retries=5 while :; do # Find the leader private IP address leader_private_ip=$($binpath status -format json | jq '.leader_address | rtrimstr(":8200") | ltrimstr("http://")') - match_ip=$(echo $instance_ips |jq -r --argjson ip $leader_private_ip 'map(select(. == $ip))') + match_ip=$(echo "$instance_ips" |jq -r --argjson ip "$leader_private_ip" 'map(select(. == $ip))') if [[ "$leader_private_ip" != 'null' ]] && [[ "$match_ip" != '[]' ]]; then echo "$leader_private_ip" | sed 's/\"//g' diff --git a/enos/modules/vault_verify_unsealed/templates/verify-vault-node-unsealed.sh b/enos/modules/vault_verify_unsealed/templates/verify-vault-node-unsealed.sh index 426963cc77..4ae3bd2a9e 100644 --- a/enos/modules/vault_verify_unsealed/templates/verify-vault-node-unsealed.sh +++ b/enos/modules/vault_verify_unsealed/templates/verify-vault-node-unsealed.sh @@ -2,24 +2,36 @@ # Copyright (c) HashiCorp, Inc. # SPDX-License-Identifier: BUSL-1.1 - set -e +# shellcheck disable=SC2154 binpath=${vault_install_dir}/vault fail() { echo "$1" 1>&2 - return 1 + exit 1 } test -x "$binpath" || fail "unable to locate vault binary at $binpath" export VAULT_ADDR='http://127.0.0.1:8200' -health_status=$(curl http://127.0.0.1:8200/v1/sys/health |jq '.') -unseal_status=$($binpath status -format json | jq -Mr --argjson expected "false" '.sealed == $expected') -if [[ "$unseal_status" != 'true' ]]; then - fail "expected ${vault_cluster_addr} to be unsealed, got unseal status: $unseal_status" -fi +count=0 +retries=4 +while :; do + health_status=$(curl http://127.0.0.1:8200/v1/sys/health |jq '.') + unseal_status=$($binpath status -format json | jq -Mr --argjson expected "false" '.sealed == $expected') + if [[ "$unseal_status" == 'true' ]]; then + echo "$health_status" + exit 0 + fi -echo $health_status + wait=$((2 ** count)) + count=$((count + 1)) + if [ "$count" -lt "$retries" ]; then + sleep "$wait" + else + # shellcheck disable=SC2154 + fail "expected ${vault_cluster_addr} to be unsealed, got unseal status: $unseal_status" + fi +done diff --git a/scripts/ci-helper.sh b/scripts/ci-helper.sh index d863b02e0d..611df09c6c 100755 --- a/scripts/ci-helper.sh +++ b/scripts/ci-helper.sh @@ -85,7 +85,7 @@ function build() { : "${GO_TAGS:=""}" : "${REMOVE_SYMBOLS:=""}" - GOOS= GOARCH= go generate ./... + (unset GOOS; unset GOARCH; go generate ./...) # Build our ldflags msg="--> Building Vault revision $revision, built $build_date" @@ -129,53 +129,10 @@ function prepare_legal() { popd } -# Determine the matrix group number that we'll select for execution. If the -# MATRIX_TEST_GROUP environment variable has set then it will always return -# that value. If has not been set, we will randomly select a number between 1 -# and the value of MATRIX_MAX_TEST_GROUPS. -function matrix_group_id() { - : "${MATRIX_TEST_GROUP:=""}" - if [ -n "$MATRIX_TEST_GROUP" ]; then - echo "$MATRIX_TEST_GROUP" - return - fi - - : "${MATRIX_MAX_TEST_GROUPS:=1}" - awk -v min=1 -v max=$MATRIX_MAX_TEST_GROUPS 'BEGIN{srand(); print int(min+rand()*(max-min+1))}' -} - -# Filter matrix file reads in the contents of MATRIX_FILE and filters out -# scenarios that are not in the current test group and/or those that have not -# met minimux or maximum version requirements. -function matrix_filter_file() { - : "${MATRIX_FILE:=""}" - if [ -z "$MATRIX_FILE" ]; then - echo "You must specify the MATRIX_FILE variable for this command" >&2 - exit 1 - fi - - : "${VAULT_MINOR_VERSION:=""}" - if [ -z "$VAULT_MINOR_VERSION" ]; then - echo "You must specify the VAULT_MINOR_VERSION variable for this command" >&2 - exit 1 - fi - - : "${MATRIX_TEST_GROUP:=$(matrix_group_id)}" - - local path - local matrix - path=$(readlink -f $MATRIX_FILE) - matrix=$(cat "$path" | jq ".include | - map(. | - select( - ((.min_minor_version == null) or (.min_minor_version <= $VAULT_MINOR_VERSION)) and - ((.max_minor_version == null) or (.max_minor_version >= $VAULT_MINOR_VERSION)) and - ((.test_group == null) or (.test_group == $MATRIX_TEST_GROUP)) - ) - )" - ) - - echo "{\"include\":$matrix}" | jq -c . +# Package version converts a vault version string into a compatible representation for system +# packages. +function version_package() { + awk '{ gsub("-","~",$1); print $1 }' <<< "$VAULT_VERSION" } # Run the CI Helper @@ -199,12 +156,6 @@ function main() { prepare-legal) prepare_legal ;; - matrix-filter-file) - matrix_filter_file - ;; - matrix-group-id) - matrix_group_id - ;; revision) build_revision ;;