From 4f58e1f110c572ead569086237bd804eea4d6757 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 30 Aug 2022 11:17:00 +0200 Subject: [PATCH 1/4] ci-automation: Break test cycle properly Create a tapfile and break out of the loop. --- ci-automation/vendor-testing/qemu_update.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ci-automation/vendor-testing/qemu_update.sh b/ci-automation/vendor-testing/qemu_update.sh index 2d8dea3a39..1ce875f684 100755 --- a/ci-automation/vendor-testing/qemu_update.sh +++ b/ci-automation/vendor-testing/qemu_update.sh @@ -13,7 +13,12 @@ set -euo pipefail source ci-automation/vendor_test.sh if [ "$*" != "" ] && [ "$*" != "*" ] && [ "$*" != "cl.update.payload" ]; then - echo "Only cl.update.payload is supported, got '$*'" + echo "1..1" > "${CIA_TAPFILE}" + echo "not ok - all qemu update tests" >> "${CIA_TAPFILE}" + echo " ---" >> "${CIA_TAPFILE}" + echo " ERROR: Only cl.update.payload is supported, got '$*'." | tee -a "${CIA_TAPFILE}" + echo " ..." >> "${CIA_TAPFILE}" + break_retest_cycle exit 1 fi From 3ce85303f67f8fe9f1ad0fe370712711682720aa Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 30 Aug 2022 11:24:32 +0200 Subject: [PATCH 2/4] ci-automation: Return 1 on broken cycle We have set success to true when the test cycle was broken, which was a hacky way to avoid printing the give up message. But this setting success to true also meant that the script returned with status 0, which is wrong. Add another variable for controlling printing the give up message. --- ci-automation/test.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ci-automation/test.sh b/ci-automation/test.sh index 1b012b60e6..13425c5734 100644 --- a/ci-automation/test.sh +++ b/ci-automation/test.sh @@ -158,6 +158,7 @@ function _test_run_impl() { local tap_merged_detailed="results-${image}-detailed.tap" local retry="" local success=false + local print_give_up=true # A job on each worker prunes old mantle images (docker image prune) echo "docker rm -f '${container_name}'" >> ./ci-cleanup.sh @@ -197,6 +198,7 @@ function _test_run_impl() { if [ -z "$failed_tests" ] ; then echo "########### All tests succeeded. ###########" success=true + print_give_up=false break fi @@ -204,9 +206,7 @@ function _test_run_impl() { echo "########### Test cycle requested to break ###########" echo "Failed tests: $failed_tests" echo "-----------" - # not really a success, but don't print a message about - # exhaused reruns and giving up - success=true + print_give_up=false break fi @@ -217,7 +217,7 @@ function _test_run_impl() { done - if ! $success; then + if ${print_give_up}; then echo "########### All re-runs exhausted ($retries). Giving up. ###########" fi From 821235754cc5d2bc56c65b0691fc4137bc034bf0 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 30 Aug 2022 11:27:10 +0200 Subject: [PATCH 3/4] ci-automation: Print failed tests nicer Instead of printing failed tests like this: Failed tests: kubeadm.v1.25.0.cilium.base kubeadm.v1.24.1.cilium.base Do it like this: Failed tests: kubeadm.v1.25.0.cilium.base kubeadm.v1.24.1.cilium.base --- ci-automation/test.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ci-automation/test.sh b/ci-automation/test.sh index 13425c5734..bbff2f3e2c 100644 --- a/ci-automation/test.sh +++ b/ci-automation/test.sh @@ -204,14 +204,16 @@ function _test_run_impl() { if retest_cycle_broken; then echo "########### Test cycle requested to break ###########" - echo "Failed tests: $failed_tests" + echo "Failed tests:" + echo "${failed_tests}" echo "-----------" print_give_up=false break fi echo "########### Some tests failed and will be re-run (${retry} / ${retries}). ###########" - echo "Failed tests: $failed_tests" + echo "Failed tests:" + echo "${failed_tests}" echo "-----------" set -- $failed_tests done From 9def5c8654f767c88278dc671b52c25a29cf6e6c Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 30 Aug 2022 12:27:59 +0200 Subject: [PATCH 4/4] ci-automation: Use an array for storing failed tests --- ci-automation/test.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ci-automation/test.sh b/ci-automation/test.sh index bbff2f3e2c..5151092be7 100644 --- a/ci-automation/test.sh +++ b/ci-automation/test.sh @@ -159,6 +159,7 @@ function _test_run_impl() { local retry="" local success=false local print_give_up=true + local failed_tests=() # A job on each worker prunes old mantle images (docker image prune) echo "docker rm -f '${container_name}'" >> ./ci-cleanup.sh @@ -193,9 +194,8 @@ function _test_run_impl() { "${tap_merged_summary}" \ "${tap_merged_detailed}" - local failed_tests - failed_tests="$(cat "${tests_dir}/${failfile}")" - if [ -z "$failed_tests" ] ; then + readarray -t failed_tests <"${tests_dir}/${failfile}" + if [ "${#failed_tests[@]}" -eq 0 ] ; then echo "########### All tests succeeded. ###########" success=true print_give_up=false @@ -205,7 +205,7 @@ function _test_run_impl() { if retest_cycle_broken; then echo "########### Test cycle requested to break ###########" echo "Failed tests:" - echo "${failed_tests}" + printf '%s\n' "${failed_tests[@]}" echo "-----------" print_give_up=false break @@ -213,9 +213,9 @@ function _test_run_impl() { echo "########### Some tests failed and will be re-run (${retry} / ${retries}). ###########" echo "Failed tests:" - echo "${failed_tests}" + printf '%s\n' "${failed_tests[@]}" echo "-----------" - set -- $failed_tests + set -- "${failed_tests[@]}" done