From 5defa54bd42b7ee1dadf656a5dc9cb371d885df8 Mon Sep 17 00:00:00 2001 From: Kuba Wieczorek Date: Fri, 7 Jul 2023 16:18:29 +0100 Subject: [PATCH] VAULT-17592 Extract failed Go test results across runners (#21625) --- .github/workflows/ci.yml | 72 ++++++++++++++++++++++++++++++++++- .github/workflows/test-go.yml | 13 ++++--- go.mod | 2 +- go.sum | 4 +- 4 files changed, 81 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7e957eb427..fcbf8b0d20 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -288,7 +288,6 @@ jobs: steps: - run: | tr -d '\n' <<< '${{ toJSON(needs.*.result) }}' | grep -q -v -E '(failure|cancelled)' - notify-tests-completed-failures-oss: if: ${{ always() && github.repository == 'hashicorp/vault' && needs.tests-completed.result == 'failure' && (github.ref_name == 'main' || startsWith(github.ref_name, 'release/')) }} runs-on: ubuntu-latest @@ -342,3 +341,74 @@ jobs: payload: | {"text":"Enterprise test failures on ${{ github.ref_name }}","blocks":[{"type":"header","text":{"type":"plain_text","text":":rotating_light: Enterprise test failures :rotating_light:","emoji":true}},{"type":"divider"},{"type":"section","text":{"type":"mrkdwn","text":"test(s) failed on ${{ github.ref_name }}"},"accessory":{"type":"button","text":{"type":"plain_text","text":"View Failing Workflow","emoji":true},"url":"${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"}}]} + test-summary: + name: Go test failures + runs-on: ubuntu-latest + if: success() || failure() || needs.tests-completed.result == 'skipped' + needs: + - test-go + - test-go-fips + - test-go-race + - tests-completed + steps: + - name: Download failure summary + uses: actions/download-artifact@v3 + with: + name: failure-summary + - name: Prepare failure summary + run: | + # We will store the jq query results in a temp file + temp_file_name=temp-$(date +%s) + + # The 'jq' command below filters and formats JSON data from input files to generate a failure summary report. + # The query is a bit of a nightmare, though, so let's have a closer look at it. + # + # The command takes input from files matching the pattern "failure-summary-*.json". + # The input files should contain streams of JSON objects(with no commas in between), + # one per line, representing test results. + # Each object should have the "Action" and "Package" keys. + # + # We invoke the command with two flags: + # - '-r' specifies that the output should be in raw format, + # without any JSON formatting. (I.e. no quotes). + # - '-n' tells 'jq' not to read any input from the command line. + # It is used when input is provided through the 'inputs' function or other methods. + # + # 'inputs': + # Read JSON objects from the input files specified after the 'jq' command. + # We assume that the input files contain one JSON object per line. + # + # 'select(.Action == "fail") | select(.Test != null)': + # Filter JSON array to contain only objects where the value of "Action" is "fail" + # and the value of "Test" key is not null. + # + # The remaining part of the query constructs a formatted string for each filtered JSON object`: + # - '\(.Package)' and '\(.Test)' insert the values of the "Package" and "Test" keys into the string, + # respectively. + # - 'input_filename' is a special variable in 'jq' that represents the name of the input file being processed. + # - 'split("-")' splits the input filename on the hyphen ("-") character and returns an array of the + # resulting parts. + # - '.sub(".json"; "")' removes the ".json" extension from the string. + # - The third part of the filename is extracted using '.split("-") | .[2]'. + # - If the fourth part of the filename exists, it contains the test type. + # Otherwise, the default value "normal" is used. + # - The '.sub(".json"; "")' removes the ".json" extension from the string. + # + # The filtered and formatted data is outputted as rows of a Markdown table, like this: + # | pkg1 | test1 | 1 | normal | + # | pkg2 | test2 | 4 | race | + # | pkg3 | test3 | 6 | fips | + + jq -r -n 'inputs | select(.Action == "fail") | select(.Test != null) | "| \(.Package) | \(.Test) | \(input_filename | split("-") | .[2] | sub(".json"; "")) | \(input_filename | split("-") | .[3] // "normal" | sub(".json";"") )"' failure-summary-*.json | sort >> "$temp_file_name" + + # if there are test failures, present them in a format of a GH Markdown table + if [ -s "$temp_file_name" ]; then + # shellcheck disable=SC2129 + # Here we create the headings for the summary table + echo "### Go test failures" >> "$GITHUB_STEP_SUMMARY" + echo "| Package | Test | Runner Index | Test Type |" >> "$GITHUB_STEP_SUMMARY" + echo "| ------- | ---- | ------------ | --------- |" >> "$GITHUB_STEP_SUMMARY" + cat "$temp_file_name" >> "$GITHUB_STEP_SUMMARY" + else + echo "### All Go tests passed! :white_check_mark:" >> "$GITHUB_STEP_SUMMARY" + fi diff --git a/.github/workflows/test-go.yml b/.github/workflows/test-go.yml index 0fc7b733a6..872c8df163 100644 --- a/.github/workflows/test-go.yml +++ b/.github/workflows/test-go.yml @@ -230,6 +230,7 @@ jobs: go run gotest.tools/gotestsum --format=short-verbose \ --junitfile test-results/go-test/results-${{ matrix.runner-index }}.xml \ --jsonfile test-results/go-test/results-${{ matrix.runner-index }}.json \ + --jsonfile-timing-events failure-summary-${{ matrix.runner-index }}${{inputs.name}}.json \ -- \ -tags "${{ inputs.go-build-tags }}" \ -timeout=${{ env.TIMEOUT_IN_MINUTES }}m \ @@ -258,10 +259,10 @@ jobs: with: name: test-results${{ inputs.name }} path: test-results/ - if: success() || failure() - - name: Create a summary of tests - uses: test-summary/action@62bc5c68de2a6a0d02039763b8c754569df99e3f # TSCCR: no entry for repository "test-summary/action" + if: success() || failure() + - name: Upload failure summary + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + if: success() || failure() with: - paths: "test-results/go-test/results-${{ matrix.runner-index }}.xml" - show: "fail" - if: success() || failure() + name: failure-summary + path: failure-summary-${{ matrix.runner-index }}${{inputs.name}}.json diff --git a/go.mod b/go.mod index 9cb9740a1d..1831ceaa8d 100644 --- a/go.mod +++ b/go.mod @@ -219,7 +219,7 @@ require ( google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 google.golang.org/protobuf v1.30.0 gopkg.in/ory-am/dockertest.v3 v3.3.4 - gotest.tools/gotestsum v1.9.0 + gotest.tools/gotestsum v1.10.0 honnef.co/go/tools v0.4.3 k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 layeh.com/radius v0.0.0-20190322222518-890bc1058917 diff --git a/go.sum b/go.sum index 5dcb8cfd4c..81a881a42d 100644 --- a/go.sum +++ b/go.sum @@ -3854,8 +3854,8 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -gotest.tools/gotestsum v1.9.0 h1:Jbo/0k/sIOXIJu51IZxEAt27n77xspFEfL6SqKUR72A= -gotest.tools/gotestsum v1.9.0/go.mod h1:6JHCiN6TEjA7Kaz23q1bH0e2Dc3YJjDUZ0DmctFZf+w= +gotest.tools/gotestsum v1.10.0 h1:lVO4uQJoxdsJb7jgmr1fg8QW7zGQ/tuqvsq5fHKyoHQ= +gotest.tools/gotestsum v1.10.0/go.mod h1:6JHCiN6TEjA7Kaz23q1bH0e2Dc3YJjDUZ0DmctFZf+w= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= gotest.tools/v3 v3.3.0/go.mod h1:Mcr9QNxkg0uMvy/YElmo4SpXgJKWgQvYrT7Kw5RzJ1A=