VAULT-12958 Add link to logs to the test failure summary in CI (#21736)

.github/workflows/ci.yml

@@ -136,7 +136,7 @@ jobs:
       go-build-tags: ${{ needs.setup.outputs.go-build-tags }}
       runs-on: ${{ needs.setup.outputs.compute-huge }}
       enterprise: ${{ needs.setup.outputs.enterprise }}
-      name: "-race"
+      name: "race"
     secrets: inherit
   test-go-fips:
     name: Run Go tests with FIPS configuration
@@ -161,14 +161,14 @@ jobs:
       go-build-tags: '${{ needs.setup.outputs.go-build-tags }},deadlock,cgo,fips,fips_140_2'
       runs-on: ${{ needs.setup.outputs.compute-larger }}
       enterprise: ${{ needs.setup.outputs.enterprise }}
-      name: "-fips"
+      name: "fips"
     secrets: inherit
   test-ui:
     name: Test UI
     # The test-ui job is only run on:
     # - pushes to main and branches starting with "release/"
     # - PRs where the branch starts with "ui/", "backport/ui/", "merge", or when base branch starts with "release/"
-    # - PRs with the "ui" label on github
+    # - PRs with the "ui" label on GitHub
     if: |
       github.ref_name == 'main' ||
       startsWith(github.ref_name, 'release/') ||
@@ -347,56 +347,16 @@ jobs:
           name: failure-summary
       - name: Prepare failure summary
         run: |
-          # We will store the jq query results in a temp file
+          # Sort all of the summary table rows and push them to a temp file.
           temp_file_name=temp-$(date +%s)
+          cat failure-summary-*.md | sort >> "$temp_file_name"
 
-          # The 'jq' command below filters and formats JSON data from input files to generate a failure summary report.
+          # If there are test failures, present them in a format of a GitHub Markdown table.
-          # The query is a bit of a nightmare, though, so let's have a closer look at it.
-          #
-          # The command takes input from files matching the pattern "failure-summary-*.json".
-          # The input files should contain streams of JSON objects(with no commas in between),
-          # one per line, representing test results.
-          # Each object should have the "Action" and "Package" keys.
-          #
-          # We invoke the command with two flags:
-          # - '-r' specifies that the output should be in raw format,
-          #   without any JSON formatting. (I.e. no quotes).
-          # - '-n' tells 'jq' not to read any input from the command line.
-          #   It is used when input is provided through the 'inputs' function or other methods.
-          #
-          # 'inputs':
-          # Read JSON objects from the input files specified after the 'jq' command.
-          # We assume that the input files contain one JSON object per line.
-          #
-          # 'select(.Action == "fail")':
-          # Filter JSON array to contain only objects where the value of "Action" is "fail"
-          #
-          # The remaining part of the query constructs a formatted string for each filtered JSON object`:
-          # - '\(.Package)' and '\(.Test // "-")' insert the values of the "Package" and "Test" keys into the string,
-          #   respectively. If Test is missing, insert a dash character instead.
-          # - 'input_filename' is a special variable in 'jq' that represents the name of the input file being processed.
-          # - 'split("-")' splits the input filename on the hyphen ("-") character and returns an array of the
-          #   resulting parts.
-          # - '.sub(".json"; "")' removes the ".json" extension from the string.
-          # - The third part of the filename is extracted using '.split("-") | .[2]'.
-          # - If the fourth part of the filename exists, it contains the test type.
-          #   Otherwise, the default value "normal" is used.
-          # - The '.sub(".json"; "")' removes the ".json" extension from the string.
-          #
-          # The filtered and formatted data is outputted as rows of a Markdown table, like this:
-          # | pkg1 | test1 | 1 | normal |
-          # | pkg2 | test2 | 4 | race |
-          # | pkg3 | test3 | 6 | fips |
-
-          jq -r -n 'inputs | select(.Action == "fail") | "| \(.Package) | \(.Test // "-") | \(input_filename | split("-") | .[2] | sub(".json"; "")) | \(input_filename | split("-") | .[3] // "normal" | sub(".json";"") )"' failure-summary-*.json | sort >> "$temp_file_name"
-
-          # if there are test failures, present them in a format of a GH Markdown table
           if [ -s "$temp_file_name" ]; then
             # shellcheck disable=SC2129
             # Here we create the headings for the summary table
-            echo "### Go test failures" >> "$GITHUB_STEP_SUMMARY"
+            echo "| Test Type | Package | Test | Elapsed | Runner Index | Logs |" >> "$GITHUB_STEP_SUMMARY"
-            echo "| Package | Test | Runner Index | Test Type |" >> "$GITHUB_STEP_SUMMARY"
+            echo "| --------- | ------- | ---- | ------- | ------------ | ---- |" >> "$GITHUB_STEP_SUMMARY"
-            echo "| ------- | ---- | ------------ | --------- |" >> "$GITHUB_STEP_SUMMARY"
             cat "$temp_file_name" >>  "$GITHUB_STEP_SUMMARY"
           else
             echo "### All Go tests passed! :white_check_mark:" >> "$GITHUB_STEP_SUMMARY"

.github/workflows/test-go.yml

@@ -245,7 +245,7 @@ jobs:
             go run gotest.tools/gotestsum --format=short-verbose \
               --junitfile test-results/go-test/results-${{ matrix.id }}.xml \
               --jsonfile test-results/go-test/results-${{ matrix.id }}.json \
-              --jsonfile-timing-events failure-summary-${{ matrix.id }}${{inputs.name}}.json \
+              --jsonfile-timing-events failure-summary-${{ matrix.id }}${{ inputs.name != '' && '-' || '' }}${{ inputs.name }}.json \
               -- \
               -tags "${{ inputs.go-build-tags }}" \
               -timeout=${{ env.TIMEOUT_IN_MINUTES }}m \
@@ -271,15 +271,65 @@ jobs:
       - name: Archive test results
         uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
         with:
-          name: test-results${{ inputs.name }}
+          name: test-results${{ inputs.name != '' && '-' || '' }}${{ inputs.name }}
           path: test-results/go-test
         if: success() || failure()
+      # GitHub Actions doesn't expose the job ID or the URL to the job execution,
+      # so we have to fetch it from the API
+      - name: Fetch job logs URL
+        uses: actions/github-script@d7906e4ad0b1822421a7e6a35d5ca353c962f410 # v6.4.1
+        if: success() || failure()
+        with:
+          retries: 3
+          script: |
+            const fs = require("fs");
+            const result = await github.rest.actions.listJobsForWorkflowRun({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                run_id: context.runId,
+            });
+
+            const prefixToSearchFor = 'Run Go tests / test-go (${{ matrix.id }}'
+            const jobData = result.data.jobs.filter(
+                (job) => job.name.startsWith(prefixToSearchFor)
+            );
+            if (
+                jobData === undefined ||
+                jobData.length == 0 ||
+                jobData[0].html_url === undefined
+            ) {
+                const msg = "Failed to fetch GHA job data.";
+                console.log(msg);
+                throw new Error(msg);
+            }
+
+            const url = jobData[0].html_url;
+            const envVarName = "GH_JOB_URL";
+            const envVar = envVarName + "=" + url;
+            const envFile = process.env.GITHUB_ENV;
+
+            fs.appendFile(envFile, envVar, (err) => {
+                if (err) throw err;
+                console.log("Successfully set " + envVarName + " to: " + url);
+            });
+      - name: Prepare failure summary
+        if: success() || failure()
+        run: |
+          # This jq query filters out successful tests, leaving only the failures.
+          # Then, it formats the results into rows of a Markdown table.
+          # An example row will resemble this:
+          # | github.com/hashicorp/vault/package | TestName | fips | 0 | 2 | [view results](github.com/link-to-logs) |
+          jq -r -n 'inputs
+          | select(.Action == "fail")
+          | "| ${{inputs.name}} | \(.Package) | \(.Test // "-") | \(.Elapsed) | ${{ matrix.id }} | [view test results :scroll:](${{ env.GH_JOB_URL }}) |"' \
+          failure-summary-${{ matrix.id }}${{ inputs.name != '' && '-' || '' }}${{inputs.name}}.json \
+          >> failure-summary-${{ matrix.id }}${{ inputs.name != '' && '-' || '' }}${{inputs.name}}.md
       - name: Upload failure summary
         uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
         if: success() || failure()
         with:
           name: failure-summary
-          path: failure-summary-${{ matrix.id }}${{inputs.name}}.json
+          path: failure-summary-${{ matrix.id }}${{ inputs.name != '' && '-' || '' }}${{inputs.name}}.md
 
   test-collect-reports:
     needs: test-go