mirrors/vault
1
0
Fork 0
mirror of https://github.com/hashicorp/vault.git synced 2025-11-28 22:21:30 +01:00
Code Issues Projects Releases Wiki Activity

VAULT-12958 Add link to logs to the test failure summary in CI (#21736)

Browse Source
This commit is contained in:
Kuba Wieczorek 2023-07-12 18:29:31 +01:00 committed by GitHub
parent bfa93fdeda
commit 5c5c2abc63
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 62 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
.github/workflows/ci.yml vendored
View File

@ -136,7 +136,7 @@ jobs:
go-build-tags: ${{ needs.setup.outputs.go-build-tags }}
runs-on: ${{ needs.setup.outputs.compute-huge }}
enterprise: ${{ needs.setup.outputs.enterprise }}
name: "-race"
name: "race"
secrets: inherit
test-go-fips:
name: Run Go tests with FIPS configuration
@ -161,14 +161,14 @@ jobs:
go-build-tags: '${{ needs.setup.outputs.go-build-tags }},deadlock,cgo,fips,fips_140_2'
runs-on: ${{ needs.setup.outputs.compute-larger }}
enterprise: ${{ needs.setup.outputs.enterprise }}
name: "-fips"
name: "fips"
secrets: inherit
test-ui:
name: Test UI
# The test-ui job is only run on:
# - pushes to main and branches starting with "release/"
# - PRs where the branch starts with "ui/", "backport/ui/", "merge", or when base branch starts with "release/"
# - PRs with the "ui" label on github
# - PRs with the "ui" label on GitHub
if: |
github.ref_name == 'main' ||
startsWith(github.ref_name, 'release/') ||
@ -347,56 +347,16 @@ jobs:
name: failure-summary
- name: Prepare failure summary
run: |
# We will store the jq query results in a temp file
# Sort all of the summary table rows and push them to a temp file.
temp_file_name=temp-$(date +%s)
cat failure-summary-*.md | sort >> "$temp_file_name"
# The 'jq' command below filters and formats JSON data from input files to generate a failure summary report.
# The query is a bit of a nightmare, though, so let's have a closer look at it.
#
# The command takes input from files matching the pattern "failure-summary-*.json".
# The input files should contain streams of JSON objects(with no commas in between),
# one per line, representing test results.
# Each object should have the "Action" and "Package" keys.
#
# We invoke the command with two flags:
# - '-r' specifies that the output should be in raw format,
# without any JSON formatting. (I.e. no quotes).
# - '-n' tells 'jq' not to read any input from the command line.
# It is used when input is provided through the 'inputs' function or other methods.
#
# 'inputs':
# Read JSON objects from the input files specified after the 'jq' command.
# We assume that the input files contain one JSON object per line.
#
# 'select(.Action == "fail")':
# Filter JSON array to contain only objects where the value of "Action" is "fail"
#
# The remaining part of the query constructs a formatted string for each filtered JSON object`:
# - '\(.Package)' and '\(.Test // "-")' insert the values of the "Package" and "Test" keys into the string,
# respectively. If Test is missing, insert a dash character instead.
# - 'input_filename' is a special variable in 'jq' that represents the name of the input file being processed.
# - 'split("-")' splits the input filename on the hyphen ("-") character and returns an array of the
# resulting parts.
# - '.sub(".json"; "")' removes the ".json" extension from the string.
# - The third part of the filename is extracted using '.split("-") | .[2]'.
# - If the fourth part of the filename exists, it contains the test type.
# Otherwise, the default value "normal" is used.
# - The '.sub(".json"; "")' removes the ".json" extension from the string.
#
# The filtered and formatted data is outputted as rows of a Markdown table, like this:
# | pkg1 | test1 | 1 | normal |
# | pkg2 | test2 | 4 | race |
# | pkg3 | test3 | 6 | fips |
jq -r -n 'inputs | select(.Action == "fail") | "| \(.Package) | \(.Test // "-") | \(input_filename | split("-") | .[2] | sub(".json"; "")) | \(input_filename | split("-") | .[3] // "normal" | sub(".json";"") )"' failure-summary-*.json | sort >> "$temp_file_name"
# if there are test failures, present them in a format of a GH Markdown table
# If there are test failures, present them in a format of a GitHub Markdown table.
if [ -s "$temp_file_name" ]; then
# shellcheck disable=SC2129
# Here we create the headings for the summary table
echo "### Go test failures" >> "$GITHUB_STEP_SUMMARY"
echo "| Package | Test | Runner Index | Test Type |" >> "$GITHUB_STEP_SUMMARY"
echo "| ------- | ---- | ------------ | --------- |" >> "$GITHUB_STEP_SUMMARY"
echo "| Test Type | Package | Test | Elapsed | Runner Index | Logs |" >> "$GITHUB_STEP_SUMMARY"
echo "| --------- | ------- | ---- | ------- | ------------ | ---- |" >> "$GITHUB_STEP_SUMMARY"
cat "$temp_file_name" >> "$GITHUB_STEP_SUMMARY"
else
echo "### All Go tests passed! :white_check_mark:" >> "$GITHUB_STEP_SUMMARY"

58
.github/workflows/test-go.yml vendored
View File

@ -245,7 +245,7 @@ jobs:
go run gotest.tools/gotestsum --format=short-verbose \
--junitfile test-results/go-test/results-${{ matrix.id }}.xml \
--jsonfile test-results/go-test/results-${{ matrix.id }}.json \
--jsonfile-timing-events failure-summary-${{ matrix.id }}${{inputs.name}}.json \
--jsonfile-timing-events failure-summary-${{ matrix.id }}${{ inputs.name != '' && '-' || '' }}${{ inputs.name }}.json \
-- \
-tags "${{ inputs.go-build-tags }}" \
-timeout=${{ env.TIMEOUT_IN_MINUTES }}m \
@ -271,15 +271,65 @@ jobs:
- name: Archive test results
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
with:
name: test-results${{ inputs.name }}
name: test-results${{ inputs.name != '' && '-' || '' }}${{ inputs.name }}
path: test-results/go-test
if: success() || failure()
# GitHub Actions doesn't expose the job ID or the URL to the job execution,
# so we have to fetch it from the API
- name: Fetch job logs URL
uses: actions/github-script@d7906e4ad0b1822421a7e6a35d5ca353c962f410 # v6.4.1
if: success() || failure()
with:
retries: 3
script: |
const fs = require("fs");
const result = await github.rest.actions.listJobsForWorkflowRun({
owner: context.repo.owner,
repo: context.repo.repo,
run_id: context.runId,
});
const prefixToSearchFor = 'Run Go tests / test-go (${{ matrix.id }}'
const jobData = result.data.jobs.filter(
(job) => job.name.startsWith(prefixToSearchFor)
);
if (
jobData === undefined ||
jobData.length == 0 ||
jobData[0].html_url === undefined
) {
const msg = "Failed to fetch GHA job data.";
console.log(msg);
throw new Error(msg);
}
const url = jobData[0].html_url;
const envVarName = "GH_JOB_URL";
const envVar = envVarName + "=" + url;
const envFile = process.env.GITHUB_ENV;
fs.appendFile(envFile, envVar, (err) => {
if (err) throw err;
console.log("Successfully set " + envVarName + " to: " + url);
});
- name: Prepare failure summary
if: success() || failure()
run: |
# This jq query filters out successful tests, leaving only the failures.
# Then, it formats the results into rows of a Markdown table.
# An example row will resemble this:
# | github.com/hashicorp/vault/package | TestName | fips | 0 | 2 | [view results](github.com/link-to-logs) |
jq -r -n 'inputs
| select(.Action == "fail")
| "| ${{inputs.name}} | \(.Package) | \(.Test // "-") | \(.Elapsed) | ${{ matrix.id }} | [view test results :scroll:](${{ env.GH_JOB_URL }}) |"' \
failure-summary-${{ matrix.id }}${{ inputs.name != '' && '-' || '' }}${{inputs.name}}.json \
>> failure-summary-${{ matrix.id }}${{ inputs.name != '' && '-' || '' }}${{inputs.name}}.md
- name: Upload failure summary
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
if: success() || failure()
with:
name: failure-summary
path: failure-summary-${{ matrix.id }}${{inputs.name}}.json
path: failure-summary-${{ matrix.id }}${{ inputs.name != '' && '-' || '' }}${{inputs.name}}.md
test-collect-reports:
needs: test-go
@ -297,4 +347,4 @@ jobs:
- run: |
ls -lhR test-results/go-test
find test-results/go-test -mindepth 1 -mtime +3 -delete
ls -lhR test-results/go-test
ls -lhR test-results/go-test