From 3d6db287465be1307f8d864172d0a65c9792604f Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 30 Aug 2022 11:17:00 +0200 Subject: [PATCH 1/4] ci-automation: Break test cycle properly Create a tapfile and break out of the loop. --- ci-automation/vendor-testing/qemu_update.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ci-automation/vendor-testing/qemu_update.sh b/ci-automation/vendor-testing/qemu_update.sh index 2d8dea3a39..1ce875f684 100755 --- a/ci-automation/vendor-testing/qemu_update.sh +++ b/ci-automation/vendor-testing/qemu_update.sh @@ -13,7 +13,12 @@ set -euo pipefail source ci-automation/vendor_test.sh if [ "$*" != "" ] && [ "$*" != "*" ] && [ "$*" != "cl.update.payload" ]; then - echo "Only cl.update.payload is supported, got '$*'" + echo "1..1" > "${CIA_TAPFILE}" + echo "not ok - all qemu update tests" >> "${CIA_TAPFILE}" + echo " ---" >> "${CIA_TAPFILE}" + echo " ERROR: Only cl.update.payload is supported, got '$*'." | tee -a "${CIA_TAPFILE}" + echo " ..." >> "${CIA_TAPFILE}" + break_retest_cycle exit 1 fi From 3f65334e80dd3609dde8dc580c16229db245496a Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 30 Aug 2022 11:24:32 +0200 Subject: [PATCH 2/4] ci-automation: Return 1 on broken cycle We have set success to true when the test cycle was broken, which was a hacky way to avoid printing the give up message. But this setting success to true also meant that the script returned with status 0, which is wrong. Add another variable for controlling printing the give up message. --- ci-automation/test.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ci-automation/test.sh b/ci-automation/test.sh index 0dd0fce80c..c49993f869 100644 --- a/ci-automation/test.sh +++ b/ci-automation/test.sh @@ -154,6 +154,7 @@ function _test_run_impl() { local tap_merged_detailed="results-${image}-detailed.tap" local retry="" local success=false + local print_give_up=true # A job on each worker prunes old mantle images (docker image prune) echo "docker rm -f '${container_name}'" >> ./ci-cleanup.sh @@ -193,6 +194,7 @@ function _test_run_impl() { if [ -z "$failed_tests" ] ; then echo "########### All tests succeeded. ###########" success=true + print_give_up=false break fi @@ -200,9 +202,7 @@ function _test_run_impl() { echo "########### Test cycle requested to break ###########" echo "Failed tests: $failed_tests" echo "-----------" - # not really a success, but don't print a message about - # exhaused reruns and giving up - success=true + print_give_up=false break fi @@ -213,7 +213,7 @@ function _test_run_impl() { done - if ! $success; then + if ${print_give_up}; then echo "########### All re-runs exhausted ($retries). Giving up. ###########" fi From 71c0724d03738507aeaa1c6e3b889fcca6207ffa Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 30 Aug 2022 11:27:10 +0200 Subject: [PATCH 3/4] ci-automation: Print failed tests nicer Instead of printing failed tests like this: Failed tests: kubeadm.v1.25.0.cilium.base kubeadm.v1.24.1.cilium.base Do it like this: Failed tests: kubeadm.v1.25.0.cilium.base kubeadm.v1.24.1.cilium.base --- ci-automation/test.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ci-automation/test.sh b/ci-automation/test.sh index c49993f869..9e5c3688de 100644 --- a/ci-automation/test.sh +++ b/ci-automation/test.sh @@ -200,14 +200,16 @@ function _test_run_impl() { if retest_cycle_broken; then echo "########### Test cycle requested to break ###########" - echo "Failed tests: $failed_tests" + echo "Failed tests:" + echo "${failed_tests}" echo "-----------" print_give_up=false break fi echo "########### Some tests failed and will be re-run (${retry} / ${retries}). ###########" - echo "Failed tests: $failed_tests" + echo "Failed tests:" + echo "${failed_tests}" echo "-----------" set -- $failed_tests done From 194d5d67c455e4d170417691d3903d5d834a4308 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 30 Aug 2022 12:27:59 +0200 Subject: [PATCH 4/4] ci-automation: Use an array for storing failed tests --- ci-automation/test.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ci-automation/test.sh b/ci-automation/test.sh index 9e5c3688de..f5d14d05bb 100644 --- a/ci-automation/test.sh +++ b/ci-automation/test.sh @@ -155,6 +155,7 @@ function _test_run_impl() { local retry="" local success=false local print_give_up=true + local failed_tests=() # A job on each worker prunes old mantle images (docker image prune) echo "docker rm -f '${container_name}'" >> ./ci-cleanup.sh @@ -189,9 +190,8 @@ function _test_run_impl() { "${tap_merged_summary}" \ "${tap_merged_detailed}" - local failed_tests - failed_tests="$(cat "${tests_dir}/${failfile}")" - if [ -z "$failed_tests" ] ; then + readarray -t failed_tests <"${tests_dir}/${failfile}" + if [ "${#failed_tests[@]}" -eq 0 ] ; then echo "########### All tests succeeded. ###########" success=true print_give_up=false @@ -201,7 +201,7 @@ function _test_run_impl() { if retest_cycle_broken; then echo "########### Test cycle requested to break ###########" echo "Failed tests:" - echo "${failed_tests}" + printf '%s\n' "${failed_tests[@]}" echo "-----------" print_give_up=false break @@ -209,9 +209,9 @@ function _test_run_impl() { echo "########### Some tests failed and will be re-run (${retry} / ${retries}). ###########" echo "Failed tests:" - echo "${failed_tests}" + printf '%s\n' "${failed_tests[@]}" echo "-----------" - set -- $failed_tests + set -- "${failed_tests[@]}" done