diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5d05d00a4c..75f0b44f5c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -136,7 +136,7 @@ jobs: go-build-tags: ${{ needs.setup.outputs.go-build-tags }} runs-on: ${{ needs.setup.outputs.compute-huge }} enterprise: ${{ needs.setup.outputs.enterprise }} - name: "-race" + name: "race" secrets: inherit test-go-fips: name: Run Go tests with FIPS configuration @@ -161,14 +161,14 @@ jobs: go-build-tags: '${{ needs.setup.outputs.go-build-tags }},deadlock,cgo,fips,fips_140_2' runs-on: ${{ needs.setup.outputs.compute-larger }} enterprise: ${{ needs.setup.outputs.enterprise }} - name: "-fips" + name: "fips" secrets: inherit test-ui: name: Test UI # The test-ui job is only run on: # - pushes to main and branches starting with "release/" # - PRs where the branch starts with "ui/", "backport/ui/", "merge", or when base branch starts with "release/" - # - PRs with the "ui" label on github + # - PRs with the "ui" label on GitHub if: | github.ref_name == 'main' || startsWith(github.ref_name, 'release/') || @@ -347,56 +347,16 @@ jobs: name: failure-summary - name: Prepare failure summary run: | - # We will store the jq query results in a temp file + # Sort all of the summary table rows and push them to a temp file. temp_file_name=temp-$(date +%s) + cat failure-summary-*.md | sort >> "$temp_file_name" - # The 'jq' command below filters and formats JSON data from input files to generate a failure summary report. - # The query is a bit of a nightmare, though, so let's have a closer look at it. - # - # The command takes input from files matching the pattern "failure-summary-*.json". - # The input files should contain streams of JSON objects(with no commas in between), - # one per line, representing test results. - # Each object should have the "Action" and "Package" keys. - # - # We invoke the command with two flags: - # - '-r' specifies that the output should be in raw format, - # without any JSON formatting. (I.e. no quotes). - # - '-n' tells 'jq' not to read any input from the command line. - # It is used when input is provided through the 'inputs' function or other methods. - # - # 'inputs': - # Read JSON objects from the input files specified after the 'jq' command. - # We assume that the input files contain one JSON object per line. - # - # 'select(.Action == "fail")': - # Filter JSON array to contain only objects where the value of "Action" is "fail" - # - # The remaining part of the query constructs a formatted string for each filtered JSON object`: - # - '\(.Package)' and '\(.Test // "-")' insert the values of the "Package" and "Test" keys into the string, - # respectively. If Test is missing, insert a dash character instead. - # - 'input_filename' is a special variable in 'jq' that represents the name of the input file being processed. - # - 'split("-")' splits the input filename on the hyphen ("-") character and returns an array of the - # resulting parts. - # - '.sub(".json"; "")' removes the ".json" extension from the string. - # - The third part of the filename is extracted using '.split("-") | .[2]'. - # - If the fourth part of the filename exists, it contains the test type. - # Otherwise, the default value "normal" is used. - # - The '.sub(".json"; "")' removes the ".json" extension from the string. - # - # The filtered and formatted data is outputted as rows of a Markdown table, like this: - # | pkg1 | test1 | 1 | normal | - # | pkg2 | test2 | 4 | race | - # | pkg3 | test3 | 6 | fips | - - jq -r -n 'inputs | select(.Action == "fail") | "| \(.Package) | \(.Test // "-") | \(input_filename | split("-") | .[2] | sub(".json"; "")) | \(input_filename | split("-") | .[3] // "normal" | sub(".json";"") )"' failure-summary-*.json | sort >> "$temp_file_name" - - # if there are test failures, present them in a format of a GH Markdown table + # If there are test failures, present them in a format of a GitHub Markdown table. if [ -s "$temp_file_name" ]; then # shellcheck disable=SC2129 # Here we create the headings for the summary table - echo "### Go test failures" >> "$GITHUB_STEP_SUMMARY" - echo "| Package | Test | Runner Index | Test Type |" >> "$GITHUB_STEP_SUMMARY" - echo "| ------- | ---- | ------------ | --------- |" >> "$GITHUB_STEP_SUMMARY" + echo "| Test Type | Package | Test | Elapsed | Runner Index | Logs |" >> "$GITHUB_STEP_SUMMARY" + echo "| --------- | ------- | ---- | ------- | ------------ | ---- |" >> "$GITHUB_STEP_SUMMARY" cat "$temp_file_name" >> "$GITHUB_STEP_SUMMARY" else echo "### All Go tests passed! :white_check_mark:" >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/test-go.yml b/.github/workflows/test-go.yml index 370b878e07..3ad1e2e30b 100644 --- a/.github/workflows/test-go.yml +++ b/.github/workflows/test-go.yml @@ -245,7 +245,7 @@ jobs: go run gotest.tools/gotestsum --format=short-verbose \ --junitfile test-results/go-test/results-${{ matrix.id }}.xml \ --jsonfile test-results/go-test/results-${{ matrix.id }}.json \ - --jsonfile-timing-events failure-summary-${{ matrix.id }}${{inputs.name}}.json \ + --jsonfile-timing-events failure-summary-${{ matrix.id }}${{ inputs.name != '' && '-' || '' }}${{ inputs.name }}.json \ -- \ -tags "${{ inputs.go-build-tags }}" \ -timeout=${{ env.TIMEOUT_IN_MINUTES }}m \ @@ -271,15 +271,65 @@ jobs: - name: Archive test results uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 with: - name: test-results${{ inputs.name }} + name: test-results${{ inputs.name != '' && '-' || '' }}${{ inputs.name }} path: test-results/go-test if: success() || failure() + # GitHub Actions doesn't expose the job ID or the URL to the job execution, + # so we have to fetch it from the API + - name: Fetch job logs URL + uses: actions/github-script@d7906e4ad0b1822421a7e6a35d5ca353c962f410 # v6.4.1 + if: success() || failure() + with: + retries: 3 + script: | + const fs = require("fs"); + const result = await github.rest.actions.listJobsForWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId, + }); + + const prefixToSearchFor = 'Run Go tests / test-go (${{ matrix.id }}' + const jobData = result.data.jobs.filter( + (job) => job.name.startsWith(prefixToSearchFor) + ); + if ( + jobData === undefined || + jobData.length == 0 || + jobData[0].html_url === undefined + ) { + const msg = "Failed to fetch GHA job data."; + console.log(msg); + throw new Error(msg); + } + + const url = jobData[0].html_url; + const envVarName = "GH_JOB_URL"; + const envVar = envVarName + "=" + url; + const envFile = process.env.GITHUB_ENV; + + fs.appendFile(envFile, envVar, (err) => { + if (err) throw err; + console.log("Successfully set " + envVarName + " to: " + url); + }); + - name: Prepare failure summary + if: success() || failure() + run: | + # This jq query filters out successful tests, leaving only the failures. + # Then, it formats the results into rows of a Markdown table. + # An example row will resemble this: + # | github.com/hashicorp/vault/package | TestName | fips | 0 | 2 | [view results](github.com/link-to-logs) | + jq -r -n 'inputs + | select(.Action == "fail") + | "| ${{inputs.name}} | \(.Package) | \(.Test // "-") | \(.Elapsed) | ${{ matrix.id }} | [view test results :scroll:](${{ env.GH_JOB_URL }}) |"' \ + failure-summary-${{ matrix.id }}${{ inputs.name != '' && '-' || '' }}${{inputs.name}}.json \ + >> failure-summary-${{ matrix.id }}${{ inputs.name != '' && '-' || '' }}${{inputs.name}}.md - name: Upload failure summary uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 if: success() || failure() with: name: failure-summary - path: failure-summary-${{ matrix.id }}${{inputs.name}}.json + path: failure-summary-${{ matrix.id }}${{ inputs.name != '' && '-' || '' }}${{inputs.name}}.md test-collect-reports: needs: test-go @@ -297,4 +347,4 @@ jobs: - run: | ls -lhR test-results/go-test find test-results/go-test -mindepth 1 -mtime +3 -delete - ls -lhR test-results/go-test \ No newline at end of file + ls -lhR test-results/go-test \ No newline at end of file